Compare commits
56
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e2d68264e6 | ||
|
|
8bf2b3f061 | ||
|
|
933ddce17a | ||
|
|
085557f02b | ||
|
|
2544bab0aa | ||
|
|
bea4a8bae0 | ||
|
|
31dc0b5322 | ||
|
|
18ca696093 | ||
|
|
c441b28b01 | ||
|
|
125e182a05 | ||
|
|
3dc8e4d98e | ||
|
|
b947d34583 | ||
|
|
862e527539 | ||
|
|
297d7eabd6 | ||
|
|
86f214bdc6 | ||
|
|
8420384555 | ||
|
|
f158717ae5 | ||
|
|
3a645d61b2 | ||
|
|
ac86240cd8 | ||
|
|
8c7f47ee71 | ||
|
|
8a9d4e94cf | ||
|
|
224345b00c | ||
|
|
6ee0947d03 | ||
|
|
b82f870350 | ||
|
|
819a262bd3 | ||
|
|
10a017a62d | ||
|
|
b82ec338c5 | ||
|
|
90ebbb469c | ||
|
|
32318eaf75 | ||
|
|
98d80620bf | ||
|
|
cec5a119d1 | ||
|
|
8744958c2f | ||
|
|
65b1e0addb | ||
|
|
a0b8427774 | ||
|
|
bd3897a7ec | ||
|
|
98039728a7 | ||
|
|
537f9ad677 | ||
|
|
6081e24e78 | ||
|
|
9779145f1a | ||
|
|
e1576f336e | ||
|
|
673f0364de | ||
|
|
90c4e55c40 | ||
|
|
2f610e0170 | ||
|
|
35598cb6fb | ||
|
|
af5003aee2 | ||
|
|
0e3223dd83 | ||
|
|
b1ac354f59 | ||
|
|
0147180a8b | ||
|
|
2d0a0b6c63 | ||
|
|
044ac04693 | ||
|
|
019a983732 | ||
|
|
8ff51b993c | ||
|
|
5d20efdbbd | ||
|
|
eed944d75f | ||
|
|
826f041d7f | ||
|
|
97d4558da0 |
@@ -94,16 +94,6 @@ inputs:
|
||||
description: If true, do not set any CXXFLAGS or LDFLAGS.
|
||||
default: false
|
||||
|
||||
# Unfortunately, "uses:" fields cannot have references to variables like
|
||||
# ${{env.MFEM_ACTIONS_VERSION}}, so the branch/tag name has to be hard coded.
|
||||
# Therefore, in the future, when updating the version of the
|
||||
# mfem/github-actions to use, we'll have to replace:
|
||||
# - all definitions of MFEM_ACTIONS_VERSION and
|
||||
# - all "uses:" fields that refer to mfem/github-actions.
|
||||
MFEM_ACTIONS_VERSION:
|
||||
description: Version (branch or tag) of the mfem/github-actions to use.
|
||||
default: v2.7
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
@@ -128,7 +118,6 @@ runs:
|
||||
echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
|
||||
echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
|
||||
echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
|
||||
echo MFEM_ACTIONS_VERSION=${{inputs.MFEM_ACTIONS_VERSION}} >> $GITHUB_ENV
|
||||
shell: bash
|
||||
|
||||
- name: Env (dir)
|
||||
|
||||
@@ -53,7 +53,7 @@ runs:
|
||||
run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
|
||||
shell: bash
|
||||
|
||||
- uses: mfem/github-actions/build-mfem@v2.7
|
||||
- uses: mfem/github-actions/build-mfem@v2.5
|
||||
if: ${{steps.debug.outputs.cache-hit != 'true'}}
|
||||
env:
|
||||
CXXFLAGS: ${{env.CXXFLAGS}}
|
||||
@@ -82,7 +82,7 @@ runs:
|
||||
run: find . -type f -name '*.o' -delete
|
||||
shell: bash
|
||||
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: build-${{inputs.par}}-${{inputs.sanitizer}}
|
||||
path: mfem/build
|
||||
|
||||
@@ -12,11 +12,6 @@
|
||||
name: 'Install MPI'
|
||||
description: 'Installs MPI and set up its environment variables'
|
||||
|
||||
inputs:
|
||||
NO_FLAGS:
|
||||
description: If true, do not set any CXXFLAGS or LDFLAGS.
|
||||
default: false
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
@@ -32,7 +27,6 @@ runs:
|
||||
shell: bash
|
||||
|
||||
- name: Env (bis)
|
||||
if: ${{ inputs.NO_FLAGS != 'true' }}
|
||||
run: |
|
||||
echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
|
||||
echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV
|
||||
|
||||
@@ -49,7 +49,7 @@ runs:
|
||||
par: ${{inputs.par}}
|
||||
sanitizer: ${{inputs.sanitizer}}
|
||||
|
||||
- uses: actions/download-artifact@v8
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: build-${{inputs.par}}-${{inputs.sanitizer}}
|
||||
path: mfem/build
|
||||
|
||||
@@ -37,14 +37,14 @@ runs:
|
||||
with:
|
||||
path: ${{env.HYPRE_DIR}}
|
||||
fail-on-cache-miss: true
|
||||
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{env.MFEM_ACTIONS_VERSION}}
|
||||
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
|
||||
|
||||
- uses: actions/cache/restore@v5 # Cache for Metis
|
||||
if: ${{inputs.par == 'true'}}
|
||||
with:
|
||||
path: ${{env.METIS_DIR}}
|
||||
fail-on-cache-miss: true
|
||||
key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}
|
||||
key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
|
||||
|
||||
- name: Hypre/Metis links
|
||||
if: ${{inputs.par == 'true'}}
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# MFEM Pull Request Review Agent Guide
|
||||
|
||||
## Purpose and scope
|
||||
Review MFEM PRs for correctness, maintainability, performance, portability, test coverage, and MFEM consistency. Use the diff and PR context; reference source files, tests, and CI results when available. Follow `CONTRIBUTING.md`, especially Developer Guidelines, PR rules, checklist, and testing.
|
||||
|
||||
## Critical review pillars
|
||||
- Correctness and numerical behavior
|
||||
- API and user-facing impact
|
||||
- Performance implications
|
||||
- Maintainability and portability
|
||||
|
||||
## Review workflow
|
||||
1. Read the PR description, linked issues, and intended behavior.
|
||||
2. Inspect the diff before commenting.
|
||||
3. Identify affected MFEM components, examples, tests, build or docs changes, and downstream APIs.
|
||||
4. Analyze the code against the critical review pillars.
|
||||
5. Compare the change against nearby code and MFEM patterns; flag unmotivated deviations.
|
||||
6. Check whether tests and documentation were updated appropriately.
|
||||
7. Review CI results and suggest actions.
|
||||
8. Produce a structured review with prioritized findings.
|
||||
9. Always limit conclusions to available evidence.
|
||||
|
||||
## MFEM-specific review checklist
|
||||
- Component-aware scope: identify the touched subsystem (FEM, solvers, preconditioners, linear algebra, mesh, examples, miniapps, build, or docs) and assess its impact against the review pillars.
|
||||
- Numerical and algorithmic behavior: assess issues in convergence, stability, tolerances, precision, iteration limits, and failure handling. If clear opportunities exist to improve the algorithmic approach, call them out with expected impact.
|
||||
- API and user-facing impact: assess backward compatibility, user-visible behavior and default changes, migration impact, deprecations, and whether documentation clearly explains user-facing API changes.
|
||||
- Data structure and memory semantics: assess ownership, lifetime, aliasing, container behavior, and device-host synchronization.
|
||||
- Parallel and serial behavior: assess whether the change preserves equivalent semantics in serial and parallel modes where applicable; if logic is currently mode-specific, check whether extension to the other mode is straightforward (clear abstractions, no hard-wired assumptions), document constraints, and call out expected behavior differences explicitly.
|
||||
- Backend and portability impact: assess likely cross-backend risks in CPU, CUDA, HIP, OCCA, RAJA, partial assembly, fallback paths, compiler compatibility, and platform assumptions.
|
||||
- Build, dependency, and configuration impact: assess CMake or make changes, optional dependency behavior, and feature-flag interactions.
|
||||
- Tests and docs alignment: check available regression or unit coverage evidence for changed behavior, and ensure docs are updated for new flags, APIs, options, or behavior changes.
|
||||
- MFEM developer-guideline fit: keep code lean, simple, general, logically separated, and portable; suggest C++17 improvements when they clearly improve safety, clarity, or maintainability.
|
||||
- New source files, examples, or miniapps: if a PR adds source/header files, verify they are properly wired into the relevant `makefile` and `CMakeLists.txt`, referenced in docs where applicable (including `doc/CodeDocumentation.dox`), and added to top-level `.gitignore` only when generated artifacts require it.
|
||||
- Changelog: verify `CHANGELOG` is updated if the PR introduces significant new features or user-facing changes.
|
||||
- MFEM conventions: use `real_t`; use `mfem::out`/`mfem::err` instead of `std::cout`/`std::cerr` in library code; flag large/binary files; if AI assistance is apparent but undisclosed, suggest following `CONTRIBUTING.md`.
|
||||
- Edge cases: if the PR touches complex or error-prone areas, suggest additional tests for edge cases, failure modes, and parallel behavior.
|
||||
|
||||
## Commenting guidelines
|
||||
- Keep comments concise, actionable, and grounded in the diff.
|
||||
- Focus on correctness, behavior changes, and user impact over style nits.
|
||||
- Be professional, concise, collaborative, technically precise, and avoid unsupported assumptions.
|
||||
|
||||
@@ -13,7 +13,7 @@ Note that some of these scripts use the shared MFEM GitHub Actions from the exte
|
||||
|
||||
<https://github.com/mfem/github-actions>
|
||||
|
||||
For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch (or tag) in the above from which the action is taken.
|
||||
For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
|
||||
|
||||
The current CI workflows are:
|
||||
|
||||
@@ -29,12 +29,16 @@ Runs a number of static repository-level sanity checks.
|
||||
|
||||
- `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
|
||||
|
||||
## `mfem-analysis.yml` (`build-analysis`)
|
||||
|
||||
Checks if the code builds and satisfies minimal requirements.
|
||||
|
||||
- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
|
||||
|
||||
## `builds-and-tests.yml`
|
||||
|
||||
Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
|
||||
|
||||
One matrix job runs `tests/scripts/gitignore` after `make test-noclean` to check generated artifacts against `.gitignore`.
|
||||
|
||||
Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
|
||||
|
||||
- `mfem/github-actions/build-hypre`
|
||||
|
||||
@@ -40,7 +40,6 @@ env:
|
||||
METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
|
||||
METIS_TOP_DIR: metis-4.0.3
|
||||
MFEM_TOP_DIR: mfem
|
||||
MFEM_ACTIONS_VERSION: v2.7
|
||||
|
||||
# Note for future improvements:
|
||||
#
|
||||
@@ -111,7 +110,6 @@ jobs:
|
||||
build-system: make
|
||||
hypre-target: int64
|
||||
precision: fp64
|
||||
gitignore-check: YES
|
||||
- os: ubuntu-latest
|
||||
target: opt
|
||||
codecov: NO
|
||||
@@ -142,10 +140,6 @@ jobs:
|
||||
|
||||
continue-on-error: ${{ matrix.enzyme && true || false }}
|
||||
|
||||
# Enable ccache for all jobs except Windows (would need sccache).
|
||||
env:
|
||||
USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
|
||||
|
||||
steps:
|
||||
# Fix 'No space left on device' errors for Ubuntu builds.
|
||||
- name: Run Actions Cleaner
|
||||
@@ -176,6 +170,20 @@ jobs:
|
||||
env
|
||||
shell: bash
|
||||
|
||||
# For info on Xcode see:
|
||||
# - https://github.com/actions/runner-images/issues/12541
|
||||
# - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
|
||||
- name: Xcode version setup (MacOS)
|
||||
if: matrix.os == 'macos-latest'
|
||||
run: |
|
||||
XCODE_PATH="/Applications/Xcode_16.4.app"
|
||||
echo "> sudo xcode-select -s ${XCODE_PATH}"
|
||||
sudo xcode-select -s ${XCODE_PATH}
|
||||
echo "> g++ -v"
|
||||
g++ -v
|
||||
echo "> clang++ -v"
|
||||
clang++ -v
|
||||
|
||||
# Only get MPI if defined for the job.
|
||||
# TODO: It would be nice to have only one step, e.g. with a dedicated
|
||||
# action, but I (@adrienbernede) don't see how at the moment.
|
||||
@@ -220,11 +228,11 @@ jobs:
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ env.HYPRE_TOP_DIR }}
|
||||
key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-${{ env.MFEM_ACTIONS_VERSION }}
|
||||
key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
|
||||
|
||||
- name: get hypre
|
||||
if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
|
||||
uses: mfem/github-actions/build-hypre@v2.7
|
||||
uses: mfem/github-actions/build-hypre@v2.5
|
||||
with:
|
||||
archive: ${{ env.HYPRE_ARCHIVE }}
|
||||
dir: ${{ env.HYPRE_TOP_DIR }}
|
||||
@@ -234,7 +242,7 @@ jobs:
|
||||
|
||||
- name: get hypre (Windows)
|
||||
if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
|
||||
uses: mfem/github-actions/build-hypre@v2.7
|
||||
uses: mfem/github-actions/build-hypre@v2.5
|
||||
with:
|
||||
archive: ${{ env.HYPRE_ARCHIVE }}
|
||||
dir: ${{ env.HYPRE_TOP_DIR }}
|
||||
@@ -250,11 +258,11 @@ jobs:
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ env.METIS_TOP_DIR }}
|
||||
key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-${{ env.MFEM_ACTIONS_VERSION }}
|
||||
key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
|
||||
|
||||
- name: install metis
|
||||
if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
|
||||
uses: mfem/github-actions/build-metis@v2.7
|
||||
uses: mfem/github-actions/build-metis@v2.5
|
||||
with:
|
||||
archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
|
||||
dir: ${{ env.METIS_TOP_DIR }}
|
||||
@@ -294,55 +302,9 @@ jobs:
|
||||
echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
|
||||
echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
|
||||
|
||||
# Restore the compiler cache (ccache). The key embeds the run id, so new
|
||||
# runs save a fresh snapshot; the restore-keys prefix warm-starts from the
|
||||
# most recent prior run (incl. the base branch for PRs).
|
||||
- name: cache ccache
|
||||
if: ${{ env.USE_CCACHE == 'true' }}
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: .ccache
|
||||
key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
|
||||
restore-keys: |
|
||||
ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
|
||||
|
||||
# Configure ccache and select how it is injected into the MFEM build:
|
||||
# - make: set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
|
||||
# runs ccache around g++ (not ccache around the mpicxx wrapper).
|
||||
# - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
|
||||
# - enzyme: wrap the brew clang++ via OMPI_CXX.
|
||||
# The chosen options are passed through build-mfem's 'config-options'
|
||||
# input (see the build step below).
|
||||
- name: configure ccache
|
||||
if: ${{ env.USE_CCACHE == 'true' }}
|
||||
run: |
|
||||
command -v ccache >/dev/null 2>&1 || {
|
||||
if [[ "${{ runner.os }}" == "Linux" ]]; then
|
||||
sudo apt-get update && sudo apt-get install -y ccache
|
||||
else
|
||||
brew install ccache
|
||||
fi
|
||||
}
|
||||
echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
|
||||
echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
|
||||
echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
|
||||
# Ignore header timestamps (restamped by each checkout) so direct mode hits.
|
||||
echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
|
||||
# Hash absolute paths relative to the workspace.
|
||||
echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
|
||||
if [[ "${{ matrix.enzyme }}" == "true" ]]; then
|
||||
echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
|
||||
elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
|
||||
echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
|
||||
else
|
||||
echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
|
||||
echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
|
||||
fi
|
||||
shell: bash
|
||||
|
||||
# MFEM build and test
|
||||
- name: build
|
||||
uses: mfem/github-actions/build-mfem@v2.7
|
||||
uses: mfem/github-actions/build-mfem@v2.5
|
||||
env:
|
||||
VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
|
||||
with:
|
||||
@@ -355,14 +317,9 @@ jobs:
|
||||
metis-dir: ${{ env.METIS_TOP_DIR }}
|
||||
mfem-dir: ${{ env.MFEM_TOP_DIR }}
|
||||
precision: ${{ matrix.precision }}
|
||||
config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
|
||||
config-options: ${{ matrix.config-opts }}
|
||||
library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
|
||||
|
||||
- name: ccache stats
|
||||
if: ${{ env.USE_CCACHE == 'true' }}
|
||||
run: ccache -s
|
||||
shell: bash
|
||||
|
||||
# Run checks (and only checks) on debug targets
|
||||
- name: checks
|
||||
if: matrix.build-system == 'make' && matrix.target == 'dbg'
|
||||
@@ -373,13 +330,7 @@ jobs:
|
||||
- name: tests
|
||||
if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
|
||||
run: |
|
||||
cd ${{ env.MFEM_TOP_DIR }}
|
||||
if [[ "${{ matrix.gitignore-check }}" == "YES" ]]; then
|
||||
make test-noclean
|
||||
else
|
||||
make test
|
||||
fi
|
||||
shell: bash
|
||||
cd ${{ env.MFEM_TOP_DIR }} && make test
|
||||
|
||||
- name: cmake checks
|
||||
if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
|
||||
@@ -424,16 +375,8 @@ jobs:
|
||||
# Code coverage (process and upload reports)
|
||||
- name: codecov
|
||||
if: matrix.codecov == 'YES'
|
||||
uses: mfem/github-actions/upload-coverage@v2.7
|
||||
uses: mfem/github-actions/upload-coverage@v2.5
|
||||
with:
|
||||
name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}
|
||||
name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
|
||||
project_dir: ${{ env.MFEM_TOP_DIR }}
|
||||
directories: "fem general linalg mesh"
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: gitignore
|
||||
if: matrix.gitignore-check == 'YES'
|
||||
run: |
|
||||
cd ${{ env.MFEM_TOP_DIR }}/tests/scripts
|
||||
./runtest gitignore
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
# LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
#
|
||||
# This file is part of the MFEM library. For more information and source code
|
||||
# availability visit https://mfem.org.
|
||||
#
|
||||
# MFEM is free software; you can redistribute it and/or modify it under the
|
||||
# terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
# CONTRIBUTING.md for details.
|
||||
---
|
||||
# A closed PR's caches can never be restored again, so delete them to free
|
||||
# space against the 10 GB per-repo cache limit.
|
||||
name: Cleanup PR caches
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
|
||||
jobs:
|
||||
cleanup:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Delete caches for the closed PR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
|
||||
run: |
|
||||
echo "Deleting caches for $PR_REF"
|
||||
while :; do
|
||||
ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
|
||||
[ -n "$ids" ] || break
|
||||
echo "$ids" | while read -r id; do
|
||||
[ -n "$id" ] || continue
|
||||
echo "Deleting cache $id"
|
||||
gh cache delete "$id" || echo " (already gone)"
|
||||
done
|
||||
done
|
||||
@@ -14,19 +14,9 @@ name: "Static Analysis"
|
||||
on:
|
||||
push:
|
||||
branches: ["master", "next"]
|
||||
paths-ignore: &docs-only-paths
|
||||
- "**/*.md"
|
||||
- "doc/**"
|
||||
- ".binder/**"
|
||||
- "CITATION.cff"
|
||||
- "LICENSE"
|
||||
- "NOTICE"
|
||||
- "CHANGELOG"
|
||||
- "INSTALL"
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: ["master"]
|
||||
paths-ignore: *docs-only-paths
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
# LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
#
|
||||
# This file is part of the MFEM library. For more information and source code
|
||||
# availability visit https://mfem.org.
|
||||
#
|
||||
# MFEM is free software; you can redistribute it and/or modify it under the
|
||||
# terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
# CONTRIBUTING.md for details.
|
||||
|
||||
name: "Build Analysis"
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
HYPRE_ARCHIVE: v2.19.0.tar.gz
|
||||
HYPRE_TOP_DIR: hypre-2.19.0
|
||||
METIS_ARCHIVE: metis-4.0.3.tar.gz
|
||||
METIS_TOP_DIR: metis-4.0.3
|
||||
COVERAGE_ENV: mfem-coverage
|
||||
|
||||
jobs:
|
||||
gitignore:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: checkout MFEM
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
path: mfem
|
||||
|
||||
- name: Get MPI (Linux)
|
||||
run: |
|
||||
sudo apt-get install openmpi-bin libopenmpi-dev
|
||||
export OMPI_MCA_rmaps_base_oversubscribe=1
|
||||
|
||||
- name: Cache Hypre Install
|
||||
id: hypre-cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ env.HYPRE_TOP_DIR }}
|
||||
key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5
|
||||
|
||||
- name: Get Hypre
|
||||
if: steps.hypre-cache.outputs.cache-hit != 'true'
|
||||
uses: mfem/github-actions/build-hypre@v2.5
|
||||
with:
|
||||
archive: ${{ env.HYPRE_ARCHIVE }}
|
||||
dir: ${{ env.HYPRE_TOP_DIR }}
|
||||
target: int32
|
||||
|
||||
- name: Cache Metis Install
|
||||
id: metis-cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ env.METIS_TOP_DIR }}
|
||||
key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
|
||||
|
||||
- name: Install Metis
|
||||
if: steps.metis-cache.outputs.cache-hit != 'true'
|
||||
uses: mfem/github-actions/build-metis@v2.5
|
||||
with:
|
||||
archive: ${{ env.METIS_ARCHIVE }}
|
||||
dir: ${{ env.METIS_TOP_DIR }}
|
||||
|
||||
# MFEM build and test
|
||||
- name: build-mfem
|
||||
uses: mfem/github-actions/build-mfem@v2.5
|
||||
with:
|
||||
os: ${{ runner.os }}
|
||||
target: opt
|
||||
codecov: NO
|
||||
mpi: par
|
||||
build-system: make
|
||||
hypre-dir: ${{ env.HYPRE_TOP_DIR }}
|
||||
metis-dir: ${{ env.METIS_TOP_DIR }}
|
||||
mfem-dir: mfem
|
||||
|
||||
- name: test (no clean)
|
||||
run: |
|
||||
cd mfem && make test-noclean
|
||||
|
||||
- name: gitignore
|
||||
run: |
|
||||
cd mfem/tests/scripts
|
||||
./runtest gitignore
|
||||
@@ -13,7 +13,6 @@ name: "Checks"
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
pull-requests: read
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -30,11 +29,6 @@ concurrency:
|
||||
# by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
|
||||
# and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
|
||||
# github.repository").
|
||||
#
|
||||
# The logic for the branch-history check is slightly different, since that check
|
||||
# also inspects the PR's labels to allow for overriding failures. In this case,
|
||||
# we run on all 'pull_request' triggers, but only run for 'push' triggers that
|
||||
# do not correspond to any open PRs.
|
||||
|
||||
jobs:
|
||||
file-headers-check:
|
||||
@@ -134,7 +128,10 @@ jobs:
|
||||
|
||||
branch-history:
|
||||
if: |
|
||||
github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
|
||||
github.ref != 'refs/heads/next' &&
|
||||
github.ref != 'refs/heads/master' &&
|
||||
(github.event_name == 'push' ||
|
||||
github.event.pull_request.head.repo.full_name != github.repository)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout mfem
|
||||
@@ -142,27 +139,7 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: check for pull request
|
||||
id: check_pr
|
||||
if: github.event_name == 'push'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
|
||||
--head "$GITHUB_REF_NAME" \
|
||||
--state open \
|
||||
--json number \
|
||||
--jq 'length > 0')
|
||||
echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: branch-history
|
||||
id: branch_history
|
||||
if: |
|
||||
(github.event_name == 'pull_request' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
steps.check_pr.outputs.pr_exists == 'false')
|
||||
continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
|
||||
'branch-history-override') }}
|
||||
run: |
|
||||
# We override origin to make sure we point to the main repo.
|
||||
# This is to have consistent test results on PRs from forks.
|
||||
@@ -170,9 +147,3 @@ jobs:
|
||||
git remote add origin https://github.com/mfem/mfem.git
|
||||
git checkout -b gh-actions-branch-history
|
||||
./config/githooks/pre-push --history
|
||||
|
||||
- name: report branch-history override
|
||||
if: steps.branch_history.outcome == 'failure'
|
||||
run: |
|
||||
echo "::warning::branch-history check failed, but the" \
|
||||
"'branch-history-override' label is set."
|
||||
|
||||
@@ -19,22 +19,18 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: ./.github/actions/sanitize/config
|
||||
with:
|
||||
NO_FLAGS: true
|
||||
- name: Cache
|
||||
id: cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{env.HYPRE_DIR}}
|
||||
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{ env.MFEM_ACTIONS_VERSION }}
|
||||
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
|
||||
- name: Setup
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/sanitize/mpi
|
||||
with:
|
||||
NO_FLAGS: true
|
||||
- name: Build
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
uses: mfem/github-actions/build-hypre@v2.7
|
||||
uses: mfem/github-actions/build-hypre@v2.5
|
||||
with:
|
||||
archive: ${{env.HYPRE_TGZ}}
|
||||
dir: ${{env.HYPRE_DIR}}
|
||||
|
||||
@@ -19,22 +19,18 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: ./.github/actions/sanitize/config
|
||||
with:
|
||||
NO_FLAGS: true
|
||||
- name: Cache
|
||||
id: cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{env.METIS_DIR}}
|
||||
key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}
|
||||
key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
|
||||
- name: Setup
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/sanitize/mpi
|
||||
with:
|
||||
NO_FLAGS: true
|
||||
- name: Build
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
uses: mfem/github-actions/build-metis@v2.7
|
||||
uses: mfem/github-actions/build-metis@v2.5
|
||||
with:
|
||||
archive: ${{env.METIS_TGZ}}
|
||||
dir: ${{env.METIS_DIR}}
|
||||
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if: ${{steps.restore.outputs.cache-hit != 'true'}}
|
||||
working-directory: mfem/build/tests/unit
|
||||
run: find . -type f -name '*.o' -delete
|
||||
- uses: actions/upload-artifact@v7
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: tests-${{inputs.par}}-${{inputs.sanitizer}}
|
||||
path: mfem/build/tests/unit/${{env.unit_tests}}
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
par: ${{inputs.par}}
|
||||
sanitizer: ${{inputs.sanitizer}}
|
||||
cache-path: mfem/build/tests/unit/${{env.unit_tests}}
|
||||
- uses: actions/download-artifact@v8
|
||||
- uses: actions/download-artifact@v4
|
||||
if: ${{steps.restore.outputs.cache-hit != 'true'}}
|
||||
with:
|
||||
name: tests-${{inputs.par}}-${{inputs.sanitizer}}
|
||||
|
||||
@@ -17,17 +17,7 @@ permissions:
|
||||
on:
|
||||
push:
|
||||
branches: ["master", "next"]
|
||||
paths-ignore: &docs-only-paths
|
||||
- "**/*.md"
|
||||
- "doc/**"
|
||||
- ".binder/**"
|
||||
- "CITATION.cff"
|
||||
- "LICENSE"
|
||||
- "NOTICE"
|
||||
- "CHANGELOG"
|
||||
- "INSTALL"
|
||||
pull_request:
|
||||
paths-ignore: *docs-only-paths
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -214,7 +214,6 @@ miniapps/adjoint/adjoint_advection_diffusion
|
||||
|
||||
miniapps/dfem/dfem-minimal-surface
|
||||
miniapps/dfem/dfem-minimal-surface-output
|
||||
miniapps/dfem/dfem-hyperelasticity_energy
|
||||
|
||||
miniapps/electromagnetics/volta
|
||||
miniapps/electromagnetics/tesla
|
||||
@@ -261,7 +260,6 @@ miniapps/meshing/polar-nc
|
||||
miniapps/meshing/mesh-quality
|
||||
miniapps/meshing/hpref
|
||||
miniapps/meshing/phpref
|
||||
miniapps/meshing/pref321
|
||||
miniapps/meshing/mobius-strip.mesh
|
||||
miniapps/meshing/klein-bottle.mesh
|
||||
miniapps/meshing/toroid-*.mesh
|
||||
|
||||
@@ -102,14 +102,12 @@ report_baseline:
|
||||
mkdir -p ${MACHINE_NAME}
|
||||
rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
|
||||
rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
|
||||
status=0
|
||||
cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
|
||||
cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
|
||||
printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
|
||||
>> ${rundir}/pipeline.txt
|
||||
# We create an autotest-email.html file, because that's how we signal
|
||||
# that there was an error / diff (temporary).
|
||||
if [[ $status -ne 0 ]] || \
|
||||
[[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
|
||||
if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
|
||||
[[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
|
||||
cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
|
||||
fi
|
||||
|
||||
@@ -85,8 +85,3 @@ opt_par_gcc_10_pumi:
|
||||
extends: .mfem_job_on_dane
|
||||
variables:
|
||||
SPEC: "%gcc@10.3.1 +pumi"
|
||||
|
||||
opt_par_gcc_10_gslib:
|
||||
extends: .mfem_job_on_dane
|
||||
variables:
|
||||
SPEC: "%gcc@10.3.1 +gslib"
|
||||
|
||||
@@ -63,8 +63,3 @@ opt_mpi_cuda_hypre_cuda_gcc:
|
||||
extends: .mfem_job_on_matrix
|
||||
variables:
|
||||
SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"
|
||||
|
||||
opt_mpi_cuda_gcc_gslib:
|
||||
extends: .mfem_job_on_matrix
|
||||
variables:
|
||||
SPEC: "%gcc@10.3.1 +mpi +cuda +gslib cuda_arch=90 ^hypre+cuda"
|
||||
|
||||
@@ -32,9 +32,9 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
|
||||
|
||||
# run
|
||||
if [[ "${MACHINE_NAME}" == "dane" ]]; then
|
||||
srun --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
|
||||
salloc --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
|
||||
elif [[ ${MACHINE_NAME} == "corona" ]]; then
|
||||
srun --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
|
||||
salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
|
||||
else
|
||||
echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
|
||||
exit 1
|
||||
|
||||
@@ -8,118 +8,43 @@
|
||||
https://mfem.org
|
||||
|
||||
|
||||
Version 4.9.1 (development)
|
||||
===========================
|
||||
|
||||
- Added policy for AI-assisted contribution to CONTRIBUTING.md.
|
||||
Version 4.10 (development)
|
||||
==========================
|
||||
|
||||
Discretization improvements
|
||||
---------------------------
|
||||
- Improved FindPointsGSLIB surface mesh capability with support for simplices
|
||||
and an option to specify axis-aligned bounding box padding for near-surface
|
||||
point queries.
|
||||
- Replaced legacy simplex quadrature rules with symmetric positive-weight
|
||||
rules for triangles (orders 0-25) and tetrahedra (orders 0-20). These
|
||||
rules guarantee all-positive weights and interior quadrature points,
|
||||
improving numerical stability. Higher orders fall back to Grundmann-Moller.
|
||||
Triangle rules: Witherden & Vincent, Comput. Math. Appl. 69(10):1232-1241,
|
||||
2015.
|
||||
Tet rules (d=1-13): Witherden & Vincent (ibid).
|
||||
Tet rules (d=14-20): Chuluunbaatar et al., Comput. Math. Appl. 124:89-97,
|
||||
2022.
|
||||
|
||||
- Added GPU-enabled partial assembly for simplicial Bernstein H1 basis based on
|
||||
ragged tensor algorithms (see DOI: 10.1137/11082539X) for mass and diffusion
|
||||
integrators.
|
||||
|
||||
- Replaced legacy simplex quadrature rules with symmetric positive weight rules
|
||||
for triangles (orders 0-25) and tetrahedra (orders 0-20). These rules
|
||||
guarantee all-positive weights and interior quadrature points, improving
|
||||
numerical stability. Higher orders fall back to Grundmann-Moller.
|
||||
* Triangle rules: Witherden and Vincent, DOI: 10.1016/j.camwa.2015.03.017
|
||||
* Tet rules (d=1-13): Witherden and Vincent (same as above)
|
||||
* Tet rules (d=14-20): Chuluunbaatar et al., DOI: 10.1016/j.camwa.2022.08.016
|
||||
Version 4.9.1 (development)
|
||||
===========================
|
||||
|
||||
- Added support for general 1D Gauss-Jacobi quadrature rules and Stroud conical
|
||||
quadrature rules on triangles and tetrahedra.
|
||||
|
||||
- Improved the GridFunction projection routines. Projections work for Scalar,
|
||||
Discretization improvements
|
||||
---------------------------
|
||||
- Improved the gridfunction projection routines. Projections work for Scalar,
|
||||
Vector and VectorFE, also NURBS versions. Optionally different types of
|
||||
projections can be selected, default behavior has not changed.
|
||||
projections can be selected, default behaviour has not changed.
|
||||
|
||||
- Added GridFunction projection methods for trace spaces, i.e., project
|
||||
coefficients on the mesh skeleton.
|
||||
|
||||
- Added methods to estimate function extremum using piecewise linear bounds plus
|
||||
- Added methods to estimate function extremum using piecewise linear bounds +
|
||||
recursive subdivision.
|
||||
|
||||
- Added class VectorQuadratureSpace that represents a vector (multi-component)
|
||||
version of the scalar (single-component) class QuadratureSpaceBase.
|
||||
|
||||
- Extend FindPointsGSLIB to support surface meshes.
|
||||
|
||||
- Added support for complex-valued mixed bilinear forms via the new classes
|
||||
MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
|
||||
SesquilinearForm classes. Rectangular complex operators are now also
|
||||
handled correctly by ComplexSparseMatrix::GetSystemMatrix and
|
||||
ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
|
||||
trial and test spaces.
|
||||
|
||||
Meshing improvements
|
||||
--------------------
|
||||
- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
|
||||
meshes with arbitrary spacing in each direction. This enables in particular
|
||||
3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
|
||||
|
||||
- Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
|
||||
bounds on the determinant of the mesh transformation Jacobian.
|
||||
|
||||
- Added PA support for TMOP's adaptive limiting functionality. Multiple
|
||||
GridFunctions and Coefficients can be combined to form a composite term.
|
||||
|
||||
- Improved support for 1D NURBS meshes with variable order, including using
|
||||
the patches construct for 1D NURBS meshes.
|
||||
|
||||
- Added the option to include material interfaces (faces separating elements
|
||||
with different element attributes) as additional boundary elements, for
|
||||
parallel visualization, e.g. with GLVis. This is supported by both the Print
|
||||
and PrintAsOne methods of ParMesh. See ParMesh::SetPrintInterfaces().
|
||||
|
||||
Linear and nonlinear solvers
|
||||
----------------------------
|
||||
- Added support for trace spaces in PRefinementTransferOperator. This is used in
|
||||
PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
|
||||
DPG miniapps).
|
||||
|
||||
GPU computing
|
||||
-------------
|
||||
- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
|
||||
kernels, kernel registration, and transpose support.
|
||||
|
||||
- Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
|
||||
memory specializations) and ElasticityIntegrator (no scratch Q-vector).
|
||||
|
||||
- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
|
||||
(AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
|
||||
|
||||
- Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
|
||||
|
||||
- Added NVIDIA cuDSS library interface. Implementation examples have been
|
||||
added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
|
||||
details. Supported versions >= 0.6.0.
|
||||
|
||||
- Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
|
||||
|
||||
- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
|
||||
partial assembly.
|
||||
|
||||
- Added support for FiniteElement::MapType::INTEGRAL spaces to
|
||||
QuadratureInterpolator.
|
||||
|
||||
New and updated examples and miniapps
|
||||
-------------------------------------
|
||||
- The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
|
||||
leverage the ParticleSet capability.
|
||||
|
||||
- Added (Complex)PRefinementMultigrid solver option in the DPG miniapps.
|
||||
|
||||
Miscellaneous
|
||||
-------------
|
||||
- Fixed signed DOF handling in ParGridFunction reading (read constructor) and
|
||||
saving via SaveAsOne(). Simplified the process of applying the DOF signs by
|
||||
using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
|
||||
method will return immediately if no sign flips are needed.
|
||||
- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
|
||||
capability.
|
||||
|
||||
|
||||
Version 4.9, released on Dec 11, 2025
|
||||
|
||||
+13
-85
@@ -184,45 +184,6 @@ if (MFEM_USE_CUDA)
|
||||
set(CMAKE_CUDA_EXTENSIONS OFF CACHE BOOL "Enable CUDA standard extensions.")
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
|
||||
find_package(CUDAToolkit REQUIRED)
|
||||
if(CMAKE_CUDA_COMPILER_ID STREQUAL "Clang")
|
||||
set(_real_fatbinary "${CMAKE_CUDA_FATBINARY}")
|
||||
set(_fatbinary_wrapper
|
||||
"${CMAKE_BINARY_DIR}/cmake-fatbinary-cuda13")
|
||||
|
||||
file(WRITE "${_fatbinary_wrapper}"
|
||||
"#!/usr/bin/env bash
|
||||
real_fatbinary=\"${_real_fatbinary}\"
|
||||
"
|
||||
[=[
|
||||
translated=()
|
||||
|
||||
for argument in "$@"; do
|
||||
case "$argument" in
|
||||
-im=profile=sm_*,file=*)
|
||||
value=${argument#-im=profile=sm_}
|
||||
architecture=${value%%,*}
|
||||
filename=${value#*,file=}
|
||||
translated+=(
|
||||
"--image3=kind=elf,sm=${architecture},file=${filename}"
|
||||
)
|
||||
;;
|
||||
*)
|
||||
translated+=("$argument")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
exec "$real_fatbinary" "${translated[@]}"
|
||||
]=])
|
||||
|
||||
file(CHMOD "${_fatbinary_wrapper}"
|
||||
PERMISSIONS
|
||||
OWNER_READ OWNER_WRITE OWNER_EXECUTE
|
||||
GROUP_READ GROUP_EXECUTE
|
||||
WORLD_READ WORLD_EXECUTE)
|
||||
|
||||
set(CMAKE_CUDA_FATBINARY "${_fatbinary_wrapper}")
|
||||
endif()
|
||||
set(CUSPARSE_FOUND TRUE)
|
||||
set(CUBLAS_FOUND TRUE)
|
||||
# Initialize CUSPARSE_LIBRARIES and CUBLAS_LIBRARIES:
|
||||
@@ -278,13 +239,6 @@ else()
|
||||
set(MFEM_DEBUG OFF)
|
||||
endif()
|
||||
|
||||
# Shadow warnings for clang only; GCC's -Wshadow flags more.
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
|
||||
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
|
||||
endif()
|
||||
|
||||
# Shared build on Windows
|
||||
if (WIN32 AND BUILD_SHARED_LIBS)
|
||||
# CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
|
||||
@@ -479,15 +433,6 @@ if (MFEM_USE_STRUMPACK)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# cuDSS can only be enabled in CUDA
|
||||
if (MFEM_USE_CUDSS)
|
||||
if (MFEM_USE_CUDA)
|
||||
find_package(CUDSS REQUIRED)
|
||||
else()
|
||||
message(FATAL_ERROR " *** cuDSS requires that CUDA be enabled.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# GnuTLS
|
||||
if (MFEM_USE_GNUTLS)
|
||||
find_package(_GnuTLS REQUIRED)
|
||||
@@ -647,13 +592,6 @@ if (MFEM_USE_ENZYME)
|
||||
set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
|
||||
endif()
|
||||
|
||||
if (MFEM_USE_PROTEUS)
|
||||
enable_language(C)
|
||||
find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
|
||||
message(STATUS "${PROTEUS_DIR}/include")
|
||||
include_directories("${PROTEUS_DIR}/include")
|
||||
endif()
|
||||
|
||||
# MFEM_TIMER_TYPE
|
||||
if (NOT DEFINED MFEM_TIMER_TYPE)
|
||||
if (APPLE)
|
||||
@@ -693,7 +631,7 @@ find_package(Threads REQUIRED)
|
||||
set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
|
||||
SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB HDF5
|
||||
NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
|
||||
ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CUDSS CALIPER CODIPACK
|
||||
ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
|
||||
BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
|
||||
ALGOIM ENZYME CUDA::cudart)
|
||||
|
||||
@@ -790,16 +728,6 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
|
||||
target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
|
||||
# message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
|
||||
target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
|
||||
|
||||
if (MFEM_USE_PROTEUS)
|
||||
add_library(ClangProteusFlags INTERFACE IMPORTED)
|
||||
set_target_properties(ClangProteusFlags PROPERTIES
|
||||
INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
|
||||
)
|
||||
target_link_libraries(mfem PUBLIC ClangProteusFlags)
|
||||
target_link_libraries(mfem PUBLIC proteus)
|
||||
endif()
|
||||
|
||||
if (TPL_TARGETS)
|
||||
add_dependencies(mfem ${TPL_TARGETS})
|
||||
endif()
|
||||
@@ -807,7 +735,7 @@ if (MINGW)
|
||||
target_link_libraries(mfem PRIVATE ws2_32)
|
||||
endif()
|
||||
if (MSVC)
|
||||
target_compile_options(mfem PUBLIC "/wd4819" "/bigobj")
|
||||
target_compile_options(mfem PUBLIC "/wd4819")
|
||||
endif()
|
||||
message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
|
||||
target_include_directories(mfem
|
||||
@@ -836,7 +764,7 @@ set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
|
||||
# If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
|
||||
# inside the build directory.
|
||||
if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
|
||||
target_compile_definitions(mfem PUBLIC
|
||||
target_compile_definitions(mfem PRIVATE
|
||||
"MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
|
||||
endif()
|
||||
|
||||
@@ -896,15 +824,16 @@ if (MFEM_ENABLE_TESTING)
|
||||
add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
|
||||
add_subdirectory(tests EXCLUDE_FROM_ALL)
|
||||
|
||||
# Create a target for all benchmarks and, optionally, enable it. Some simple
|
||||
# timer-based benchmarks in tests/benchmarks do not require Google Benchmark.
|
||||
set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
|
||||
add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
|
||||
${MFEM_ENABLE_BENCHMARKS})
|
||||
if (MFEM_ENABLE_BENCHMARKS)
|
||||
add_subdirectory(tests/benchmarks) #install benchmarks if enabled
|
||||
else()
|
||||
add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
|
||||
if (MFEM_USE_BENCHMARK)
|
||||
# Create a target for all benchmarks and, optionally, enable it.
|
||||
set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
|
||||
add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
|
||||
${MFEM_ENABLE_BENCHMARKS})
|
||||
if (MFEM_ENABLE_BENCHMARKS)
|
||||
add_subdirectory(tests/benchmarks) #install benchmarks if enabled
|
||||
else()
|
||||
add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -1084,7 +1013,6 @@ install(EXPORT ${PROJECT_NAME_UC}Targets
|
||||
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
|
||||
DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Create 'config.mk' from 'config.mk.in' for the build and install locations and
|
||||
# define install rules for 'config.mk' and 'test.mk'
|
||||
|
||||
+65
-72
@@ -3,12 +3,12 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-blue.svg"></a>
|
||||
<a href="https://github.com/mfem/mfem/releases/latest"><img alt="GitHub release" src="https://img.shields.io/github/v/release/mfem/mfem"></a>
|
||||
<a href="https://github.com/mfem/mfem/actions/workflows/repo-check.yml?query=branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
|
||||
<a href="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml?query=branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
|
||||
<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
|
||||
<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
|
||||
<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
|
||||
<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
|
||||
<a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
|
||||
<a href="https://docs.mfem.org/html/index.html"><img alt="Documentation" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
|
||||
<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
|
||||
</p>
|
||||
|
||||
|
||||
@@ -24,14 +24,6 @@ must be made under this license.
|
||||
Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
|
||||
in the MFEM community, you agree to abide by its rules.
|
||||
|
||||
## AI Policy
|
||||
- Use of AI code generation in MFEM is allowed but must be disclosed, e.g. by
|
||||
selecting the `AI-assisted` label on the PR.
|
||||
- By submitting a PR, the author acknowledges that they have reviewed and
|
||||
understand the changes they are proposing.
|
||||
- PR authors are still responsible for correctness, licensing, and attribution
|
||||
of all changes.
|
||||
|
||||
If you plan on contributing to MFEM, consider reviewing the
|
||||
[issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
|
||||
already exists for your desired feature or the bug you ran into. Use a pull
|
||||
@@ -84,7 +76,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
|
||||
follow the [MFEM PR Rules](#mfem-pr-rules).
|
||||
- When your contribution is fully working and ready to be reviewed, add
|
||||
the `ready-for-review` label.
|
||||
- PRs are treated similarly to journal submission, with an "editor" assigning two
|
||||
- PRs are treated similarly to journal submission with an "editor" assigning two
|
||||
reviewers to evaluate the changes.
|
||||
- The reviewers have 3 weeks to evaluate the PR and work with the author to
|
||||
fix issues and implement improvements.
|
||||
@@ -125,7 +117,7 @@ The MFEM source code has the following structure:
|
||||
│ ├── petsc
|
||||
│ ├── pumi
|
||||
│ ├── sundials
|
||||
│ └── superlu
|
||||
| └── superlu
|
||||
├── fem
|
||||
│ ├── ceed
|
||||
│ ├── dfem
|
||||
@@ -137,6 +129,10 @@ The MFEM source code has the following structure:
|
||||
│ ├── moonolith
|
||||
│ ├── qinterp
|
||||
│ └── tmop
|
||||
│ | ├── assemble
|
||||
│ | ├── metrics
|
||||
│ | ├── mult
|
||||
│ | └── tools
|
||||
├── general
|
||||
├── linalg
|
||||
│ ├── batched
|
||||
@@ -149,10 +145,11 @@ The MFEM source code has the following structure:
|
||||
│ ├── common
|
||||
│ ├── contact
|
||||
│ ├── dfem
|
||||
│ ├── diag-smoothers
|
||||
│ ├── dpg
|
||||
│ ├── electromagnetics
|
||||
│ ├── fluids
|
||||
│ │ ├── navier
|
||||
│ │ └── schrodinger-flow
|
||||
│ ├── gslib
|
||||
│ ├── hdiv-linear-solver
|
||||
│ ├── hooke
|
||||
@@ -162,7 +159,6 @@ The MFEM source code has the following structure:
|
||||
│ ├── nurbs
|
||||
│ ├── parelag
|
||||
│ ├── performance
|
||||
│ ├── plasma
|
||||
│ ├── shifted
|
||||
│ ├── solvers
|
||||
│ ├── spde
|
||||
@@ -193,15 +189,15 @@ respectively.
|
||||
|
||||
- The main finite element classes are:
|
||||
+ [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
|
||||
+ [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElementCollection.html)
|
||||
+ [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
|
||||
+ [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
|
||||
+ [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
|
||||
+ [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
|
||||
+ [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
|
||||
+ [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
|
||||
|
||||
- The main linear algebra classes and sources are
|
||||
+ [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
|
||||
+ [`Vector`](https://docs.mfem.org/html/classmfem_1_1Vector.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
|
||||
+ [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
|
||||
+ [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
|
||||
+ Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)
|
||||
|
||||
@@ -213,8 +209,8 @@ shared geometric entities between different tasks. The parallel source files
|
||||
have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
|
||||
|
||||
- The main parallel classes are
|
||||
+ [`ParMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
|
||||
+ [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParNCMesh.html)
|
||||
+ [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
|
||||
+ [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
|
||||
+ [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
|
||||
+ [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
|
||||
+ [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
|
||||
@@ -224,14 +220,14 @@ have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
|
||||
#### GPU and general device support
|
||||
|
||||
GPU and multi-core CPU support is based on device kernels supporting different
|
||||
backends (CUDA, HIP, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
|
||||
backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
|
||||
device/host memory manager.
|
||||
|
||||
- The main device-relevant classes and sources are:
|
||||
+ [`Device`](https://docs.mfem.org/html/device_8hpp.html)
|
||||
+ [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
|
||||
+ the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
|
||||
+ the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html), [`hip.hpp`](https://docs.mfem.org/html/hip_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
|
||||
+ the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
|
||||
|
||||
#### Utilities, building and documentation
|
||||
- The `general/` directory contains C++ classes that serve as utilities for
|
||||
@@ -245,8 +241,8 @@ device/host memory manager.
|
||||
- `examples` and `miniapps` respectively gather simple and more fully-featured
|
||||
demonstrations of the usage on MFEM. They both rely on `data/` for the
|
||||
collection of meshes.
|
||||
- The `tests/` directory contains a unit test suite, additional tests, and
|
||||
benchmarks.
|
||||
- The `tests/` directory contains a unit test suite and will later contain more
|
||||
tests that run example codes.
|
||||
|
||||
See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
|
||||
website.
|
||||
@@ -280,8 +276,8 @@ Before you can start, you need a GitHub account, here are a few suggestions:
|
||||
the top of https://github.com/mfem.
|
||||
- Consider making your membership public by going to https://github.com/orgs/mfem/people
|
||||
and clicking on the organization visibility drop box next to your name.
|
||||
- Project discussions and announcements will be posted at https://github.com/orgs/mfem/discussions,
|
||||
tagging the `@mfem/everyone` team when appropriate.
|
||||
- Project discussions and announcements will be posted at
|
||||
https://github.com/orgs/mfem/teams/everyone.
|
||||
|
||||
#### Structure
|
||||
- The MFEM source code is in the [mfem](https://github.com/mfem/mfem)
|
||||
@@ -341,12 +337,11 @@ Before you can start, you need a GitHub account, here are a few suggestions:
|
||||
- Well-designed simple code is frequently more general and powerful.
|
||||
- Lean code base is easier to understand by new collaborators.
|
||||
- New features should be added only if they are necessary or generally useful.
|
||||
- Introduction of language constructs not currently used in MFEM should be
|
||||
- Introduction of language constructions not currently used in MFEM should be
|
||||
justified and generally avoided (to maintain portability to various systems
|
||||
and compilers, including early access hardware).
|
||||
- We prefer basic C++. Use C++17 features judiciously, prioritizing readability,
|
||||
consistency with existing MFEM code, and portability to different systems,
|
||||
compilers and device backends.
|
||||
- We prefer basic C++ and the C++03 standard, to keep the code readable by
|
||||
a large audience and to make sure it compiles anywhere.
|
||||
|
||||
- *Keep the code general and reasonably efficient*
|
||||
- The main goal is fast prototyping for research and application development.
|
||||
@@ -389,7 +384,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
|
||||
- When your branch is ready for other developers to review / comment on
|
||||
the code, create a pull request towards `mfem:master`.
|
||||
|
||||
- Pull requests typically have titles like:
|
||||
- Pull request typically have titles like:
|
||||
|
||||
`Description [new-feature-dev]`
|
||||
|
||||
@@ -410,12 +405,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
|
||||
- Add a description, appropriate labels and assign yourself to the PR. The MFEM
|
||||
team will add reviewers as appropriate.
|
||||
|
||||
- List outstanding TODO items in the description.
|
||||
- List outstanding TODO items in the description, see PR #222 for an example.
|
||||
|
||||
- When your contribution is fully working and ready to be reviewed, add
|
||||
or request the `ready-for-review` label.
|
||||
the `ready-for-review` label.
|
||||
|
||||
- PRs are treated similarly to journal submission, with an "editor" assigning
|
||||
- PRs are treated similarly to journal submission with an "editor" assigning
|
||||
two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
|
||||
the PR and work with the author to implement improvements and fix issues.
|
||||
|
||||
@@ -441,7 +436,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
|
||||
checks in GitHub Actions enforce MFEM-specific rules which are explained in
|
||||
the error messages and the `tests/scripts` directory.
|
||||
|
||||
- Also note that the tests `branch-history` and `repo-check` found in GitHub
|
||||
- Also note that the tests `branch-history` and `repos-checks` found in GitHub
|
||||
Actions can be triggered automatically before each push using git hooks. See
|
||||
the [git hooks README](config/githooks/README.md) for a detailed explanation.
|
||||
|
||||
@@ -498,15 +493,15 @@ Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their a
|
||||
|
||||
3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
|
||||
|
||||
4. To seek help from the editors in case of difficulties.
|
||||
3. To seek help from the editors in case of difficulties.
|
||||
|
||||
5. To complete the review in a timely manner: 3 weeks from assignment.
|
||||
4. To complete the review in a timely manner: 3 weeks from assignment.
|
||||
|
||||
6. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
|
||||
5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
|
||||
|
||||
7. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
|
||||
6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
|
||||
|
||||
8. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
|
||||
7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
|
||||
|
||||
#### Responsibilities of Authors
|
||||
|
||||
@@ -532,30 +527,30 @@ Before a PR can be merged, it should satisfy the following:
|
||||
- [ ] Code builds.
|
||||
- [ ] Code passes `make style`.
|
||||
- [ ] Update `CHANGELOG`:
|
||||
- [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
|
||||
- [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
|
||||
- [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
|
||||
- [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
|
||||
- [ ] Update `INSTALL`:
|
||||
- [ ] Has a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
|
||||
- [ ] Have the version ranges for any required or optional libraries changed?
|
||||
- [ ] Does `make` or `cmake` have a new target?
|
||||
- [ ] Did the requirements or the installation process change? *(rare)*
|
||||
- [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
|
||||
- [ ] Have the version ranges for any required or optional libraries changed?
|
||||
- [ ] Does `make` or `cmake` have a new target?
|
||||
- [ ] Did the requirements or the installation process change? *(rare)*
|
||||
- [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
|
||||
- [ ] `.github`
|
||||
- [ ] `.appveyor.yml`
|
||||
- [ ] `.github`
|
||||
- [ ] `.appveyor.yml`
|
||||
- [ ] Update `.gitignore`:
|
||||
- [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
|
||||
- [ ] Add new patterns (just for the new files above) and re-run the above test.
|
||||
- [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
|
||||
- [ ] Add new patterns (just for the new files above) and re-run the above test.
|
||||
- [ ] New examples:
|
||||
- [ ] All sample runs at the top of the example source file work.
|
||||
- [ ] Update `examples/makefile`:
|
||||
- [ ] All sample runs at the top of the example source file work.
|
||||
- [ ] Update `examples/makefile`:
|
||||
- [ ] Add the example code to the appropriate `SEQ_EXAMPLES` and `PAR_EXAMPLES` variables.
|
||||
- [ ] Add any files generated by it to the `clean` target.
|
||||
- [ ] Add the example binary and any files generated by it to the top-level `.gitignore` file.
|
||||
- [ ] Update `examples/CMakeLists.txt`:
|
||||
- [ ] Update `examples/CMakeLists.txt`:
|
||||
- [ ] Add the example code to the `ALL_EXE_SRCS` variable.
|
||||
- [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
|
||||
- [ ] List the new example in `doc/CodeDocumentation.dox`.
|
||||
- [ ] If new examples directory (e.g. `examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
|
||||
- [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
|
||||
- [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
|
||||
- [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
|
||||
- [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
|
||||
@@ -572,13 +567,13 @@ Before a PR can be merged, it should satisfy the following:
|
||||
- [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
|
||||
- [ ] Consider adding a new test for the new miniapp.
|
||||
- [ ] List the new miniapp in `doc/CodeDocumentation.dox`
|
||||
- [ ] If new miniapps directory (e.g. `miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
|
||||
- [ ] If new miniapps directory (e.g. `miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
|
||||
- [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
|
||||
- [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
|
||||
- [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
|
||||
- [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
|
||||
- [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
|
||||
- [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
|
||||
- [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
|
||||
- [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
|
||||
- [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
|
||||
- [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
|
||||
- [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
|
||||
- [ ] New capability:
|
||||
- [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
|
||||
- [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
|
||||
@@ -680,7 +675,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
|
||||
- [ ] Update URL shortlinks:
|
||||
- [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
|
||||
- [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
|
||||
- [ ] Add the new shortlinks to the MFEM package in `spack`.
|
||||
- [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
|
||||
- [ ] Update website in `mfem/web` repo:
|
||||
- Update version and shortlinks in `src/index.md` and `src/download.md`.
|
||||
- Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.
|
||||
@@ -732,24 +727,22 @@ commit or push, see the [README](config/githooks/README.md) in the `config/githo
|
||||
directory.
|
||||
|
||||
|
||||
### GitHub Actions smoke tests
|
||||
|
||||
### Linux and Mac smoke tests
|
||||
We use GitHub Actions to drive the default tests on the `master` and `next`
|
||||
branches. See the `.github/workflows` files and the logs at
|
||||
[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
|
||||
|
||||
GitHub Actions testing should be kept lightweight, as there is a time
|
||||
constraint on jobs. The current workflows cover Linux, macOS, and Windows
|
||||
configurations.
|
||||
Testing using GitHub Actions should be kept lightweight, as there is a time
|
||||
constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
|
||||
|
||||
- Tests on the `master` branch are triggered whenever a PR is issued on this branch.
|
||||
- Tests on the `next` branch are currently scheduled to run each night.
|
||||
|
||||
### Additional Windows smoke test
|
||||
|
||||
We also use Appveyor to test building with the MS Visual C++ compiler in a Windows
|
||||
environment, as well as to test the CMake build. See the `.appveyor.yml` file
|
||||
and the build logs at
|
||||
### Windows smoke test
|
||||
We use Appveyor to test building with the MS Visual C++ compiler in a Windows
|
||||
environment, as well as to test the CMake build. See the `.appveyor` file and the
|
||||
build logs at
|
||||
[https://ci.appveyor.com/project/mfem/mfem](https://ci.appveyor.com/project/mfem/mfem).
|
||||
|
||||
CMake is used to generate the MSVC Project files and drive the build. A release
|
||||
|
||||
@@ -38,13 +38,14 @@ the option MFEM_USE_METIS.
|
||||
MFEM also includes support for devices such as GPUs, and programming models such
|
||||
as CUDA, HIP, OCCA, OpenMP and RAJA.
|
||||
|
||||
- Starting with version 4.9, MFEM requires a C++17 compiler.
|
||||
- Starting with version 4.0, MFEM requires a C++11 compiler. We recommend using
|
||||
a newer compiler, e.g. GCC version 4.9 or higher.
|
||||
|
||||
- CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
|
||||
https://developer.nvidia.com/cuda-toolkit
|
||||
|
||||
- HIP support requires an AMD GPU and an installation of the ROCm software stack
|
||||
https://rocm.docs.amd.com
|
||||
https://rocmdocs.amd.com
|
||||
|
||||
- OCCA support requires the OCCA library
|
||||
https://libocca.org
|
||||
@@ -82,9 +83,9 @@ Serial build:
|
||||
Parallel build:
|
||||
(download hypre and METIS 4 from above URLs)
|
||||
(build METIS 4 in ../metis-4.0 relative to mfem/)
|
||||
(for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
|
||||
(build hypre in ../hypre relative to mfem/)
|
||||
make parallel -j 4
|
||||
(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
|
||||
|
||||
CUDA build:
|
||||
make cuda -j 4
|
||||
@@ -114,14 +115,14 @@ Serial build:
|
||||
Parallel build:
|
||||
(download hypre and METIS 4 from above URLs)
|
||||
(build METIS 4 in ../metis-4.0 relative to mfem/)
|
||||
(for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
|
||||
(build hypre in ../hypre relative to mfem/)
|
||||
mkdir <mfem-build-dir> ; cd <mfem-build-dir>
|
||||
cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
|
||||
make -j 4
|
||||
(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
|
||||
|
||||
Parallel build with fetching of hypre and METIS:
|
||||
mkdir <mfem-build-dir> ; cd <mfem-build-dir>
|
||||
mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
|
||||
cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES
|
||||
make -j 4
|
||||
|
||||
@@ -133,8 +134,7 @@ CUDA build:
|
||||
|
||||
HIP build:
|
||||
mkdir <mfem-build-dir> ; cd <mfem-build-dir>
|
||||
cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 \
|
||||
-DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
|
||||
cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
|
||||
make -j 4
|
||||
|
||||
Example codes (serial/parallel, depending on the build):
|
||||
@@ -269,7 +269,6 @@ Compilers:
|
||||
CXX - C++ compiler, serial build
|
||||
MPICXX - MPI C++ compiler, parallel build
|
||||
CUDA_CXX - The CUDA compiler, 'nvcc' or 'clang++'
|
||||
HIP_CXX - The HIP compiler, e.g. 'hipcc'
|
||||
|
||||
Compiler options:
|
||||
OPTIM_FLAGS - Options for optimized build
|
||||
@@ -396,11 +395,6 @@ MFEM_USE_STRUMPACK = YES/NO
|
||||
classes. When enabled, this option uses the STRUMPACK_* library options, see
|
||||
below.
|
||||
|
||||
MFEM_USE_CUDSS = YES/NO
|
||||
Enable MFEM functionality based on the cuDSS library. When using cuDSS, CUDA
|
||||
support must be also enabled in MFEM, i.e. MFEM_USE_CUDA=YES must be set.
|
||||
When enabled, this option uses the CUDSS_* library options, see below.
|
||||
|
||||
MFEM_USE_GINKGO = YES/NO
|
||||
Enable MFEM functionality based on the Ginkgo library, which provides
|
||||
iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
|
||||
@@ -560,13 +554,13 @@ MFEM_USE_RAJA = YES/NO
|
||||
MFEM_USE_OCCA = YES/NO
|
||||
Enables support for the OCCA library in MFEM. OCCA is an open-source library
|
||||
which aims to make it easy to program different types of devices (e.g. CPU,
|
||||
GPU, FPGA) by providing a unified API for interacting with JIT-compiled
|
||||
GPU, FPGA) by providing an unified API for interacting with JIT-compiled
|
||||
backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
|
||||
in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.
|
||||
|
||||
MFEM_USE_GSLIB = YES/NO
|
||||
Enables MFEM functionality based on the GSLIB library, and specifically its
|
||||
FindPoints component, which provides robust algorithms to evaluate finite
|
||||
FindPoints component, which provides a robust algorithms to evaluate finite
|
||||
element functions in a collection of points in physical space. When enabled,
|
||||
the user can use the GSLIB-FindPoints methods as shown in miniapps/gslib.
|
||||
|
||||
@@ -725,18 +719,9 @@ The specific libraries and their options are:
|
||||
Options: STRUMPACK_OPT, STRUMPACK_LIB.
|
||||
Versions: STRUMPACK >= 3.0.0.
|
||||
|
||||
- CUDSS (optional), used when MFEM_USE_CUDSS = YES. Note that CUDSS requires
|
||||
CUDA 12.x toolkit and the cuDSS libraries. The supported communication backend
|
||||
is OpenMPI 4.x (default), and OpenMPI 4.x or a later version must be pre-built.
|
||||
The source files in the cuDSS tarball provide guidance for developing custom
|
||||
MPI implementations.
|
||||
URL: https://developer.nvidia.com/cudss
|
||||
https://docs.nvidia.com/cuda/cudss/advanced_features.html#communication-layer-library-in-cudss
|
||||
Options: CUDSS_OPT, CUDSS_LIB.
|
||||
Versions: cuDSS >= 0.6.0.
|
||||
|
||||
- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Ginkgo may have additional
|
||||
requirements and module-specific dependencies; see the webpage below.
|
||||
- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
|
||||
C++ compiler that supports the C++-17 standard. For additional requirements
|
||||
and dependencies of specific modules, see the Ginkgo webpage below.
|
||||
URL: https://ginkgo-project.github.io
|
||||
Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
|
||||
Debug).
|
||||
@@ -808,7 +793,7 @@ The specific libraries and their options are:
|
||||
Options: CONDUIT_OPT, CONDUIT_LIB.
|
||||
Versions: Conduit >= 0.3.1.
|
||||
|
||||
- ADIOS2 (optional), used when MFEM_USE_ADIOS2 = YES.
|
||||
- ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
|
||||
URL: https://adios2.readthedocs.io/
|
||||
Versions: ADIOS >= 2.5.0.
|
||||
|
||||
@@ -884,7 +869,7 @@ The specific libraries and their options are:
|
||||
Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
|
||||
Versions: RAJA >= 2022.10.3.
|
||||
|
||||
- Moonolith (optional), used when MFEM_USE_MOONOLITH = YES.
|
||||
- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
|
||||
URL: https://bitbucket.org/zulianp/par_moonolith
|
||||
Options: MOONOLITH_DIR
|
||||
Versions: MOONOLITH >= 1.1.0.
|
||||
@@ -972,7 +957,7 @@ CMAKE_BUILD_TYPE which can be set to standard values like "Debug", and "Release"
|
||||
To use a specific generator use the "-G <generator>" option of cmake:
|
||||
|
||||
cmake <mfem-source-dir> -G "Xcode"
|
||||
cmake <mfem-source-dir> -G "Visual Studio 17 2022"
|
||||
cmake <mfem-source-dir> -G "Visual Studio 12 2013"
|
||||
cmake <mfem-source-dir> -G "MinGW Makefiles"
|
||||
|
||||
With CMake it is possible to build MFEM as a shared library using the standard
|
||||
@@ -1217,7 +1202,7 @@ larger problems, there are two options:
|
||||
Specific options for HIP
|
||||
========================
|
||||
MFEM expects the `ROCM_PATH` environment variable to be set to the path of the
|
||||
ROCm install, as well as having `$ROCM_PATH/bin` in `PATH`.
|
||||
ROCM install, as well as having `$ROCM_PATH/bin` in `PATH`.
|
||||
|
||||
Specific options for RAJA+HIP+MPI
|
||||
=================================
|
||||
|
||||
@@ -28,7 +28,6 @@ license files. These software products and their licenses are as follows:
|
||||
* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
|
||||
* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
|
||||
* Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
|
||||
* gslib (fem/gslib.{cpp,hpp}, mesh/bb_grid_map.{cpp,hpp}) -- BSD 3-clause license
|
||||
* Picojson (fem/picojson.h) -- Custom 2-clause license
|
||||
* TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
|
||||
* Zstr (general/zstr.hpp) -- MIT license
|
||||
|
||||
@@ -35,7 +35,6 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
|
||||
set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
|
||||
set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
|
||||
set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
|
||||
set(MFEM_USE_CUDSS @MFEM_USE_CUDSS@)
|
||||
set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
|
||||
set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
|
||||
set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
|
||||
@@ -110,10 +109,6 @@ if (MFEM_USE_RAJA)
|
||||
find_dependency(RAJA)
|
||||
endif()
|
||||
|
||||
if (MFEM_USE_CUDSS)
|
||||
find_dependency(cudss)
|
||||
endif (MFEM_USE_CUDSS)
|
||||
|
||||
if (MFEM_USE_UMPIRE)
|
||||
find_dependency(umpire)
|
||||
endif()
|
||||
|
||||
@@ -108,15 +108,6 @@
|
||||
// Enable MFEM functionality based on the STRUMPACK library.
|
||||
#cmakedefine MFEM_USE_STRUMPACK
|
||||
|
||||
// Enable MFEM functionality based on the cuDSS library.
|
||||
#cmakedefine MFEM_USE_CUDSS
|
||||
|
||||
// CUDSS communication layer library path
|
||||
#cmakedefine MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
|
||||
|
||||
// CUDSS threading layer library path
|
||||
#cmakedefine MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
|
||||
|
||||
// Enable functionality based on the Ginkgo library.
|
||||
#cmakedefine MFEM_USE_GINKGO
|
||||
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
if (NOT cudss_DIR AND CUDSS_DIR)
|
||||
set(cudss_DIR ${CUDSS_DIR}/lib/cmake/cudss)
|
||||
endif()
|
||||
message(STATUS "Looking for CUDSS ...")
|
||||
message(STATUS " in CUDSS_DIR = ${CUDSS_DIR}")
|
||||
message(STATUS " cudss_DIR = ${cudss_DIR}")
|
||||
find_package(cudss)
|
||||
set(CUDSS_FOUND ${cudss_FOUND})
|
||||
set(CUDSS_LIBRARIES "cudss")
|
||||
if (CUDSS_FOUND)
|
||||
message(STATUS
|
||||
"Found CUDSS target: ${CUDSS_LIBRARIES} (version: ${cudss_VERSION})")
|
||||
else()
|
||||
set(msg STATUS)
|
||||
if (CUDSS_FIND_REQUIRED)
|
||||
set(msg FATAL_ERROR)
|
||||
endif()
|
||||
message(${msg}
|
||||
"CUDSS not found. Please set CUDSS_DIR to the install prefix.")
|
||||
endif()
|
||||
|
||||
if(CUDSS_FOUND AND TARGET cudss)
|
||||
get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION)
|
||||
if(NOT CUDSS_LIBRARY_LOCATION)
|
||||
get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION_RELEASE)
|
||||
endif()
|
||||
if(CUDSS_LIBRARY_LOCATION)
|
||||
get_filename_component(CUDSS_LIBRARY_DIR "${CUDSS_LIBRARY_LOCATION}" DIRECTORY)
|
||||
else()
|
||||
message(WARNING "Could not determine the location of the cuDSS library.")
|
||||
endif()
|
||||
else()
|
||||
message(WARNING "cuDSS target not available; cannot determine library directory.")
|
||||
endif()
|
||||
|
||||
# Set the full name of the cuDSS threading library if OpenMP is enabled.
|
||||
# The threading layer library (libcudss_mtlayer_gomp.so) is located under the
|
||||
# cuDSS library directory by default.
|
||||
if (MFEM_USE_OPENMP)
|
||||
find_file(
|
||||
CUDSS_THREADING_LIB
|
||||
NAMES libcudss_mtlayer_gomp.so
|
||||
PATHS ${CUDSS_LIBRARY_DIR}
|
||||
NO_DEFAULT_PATH
|
||||
)
|
||||
if (NOT DEFINED MFEM_CUDSS_THREADING_LIB AND CUDSS_THREADING_LIB)
|
||||
set(MFEM_CUDSS_THREADING_LIB "${CUDSS_THREADING_LIB}")
|
||||
endif()
|
||||
message(STATUS "CUDSS threading layer library: ${MFEM_CUDSS_THREADING_LIB}")
|
||||
endif()
|
||||
|
||||
# Set the full name of the cuDSS communication library if MFEM use OpenMPI.
|
||||
# The communication layer library (libcudss_commlayer_mpi.so) is located under the
|
||||
# cuDSS library directory by default.
|
||||
# The communication layer library is used pre-built communication layers for OpenMPI
|
||||
# by default.
|
||||
if (MFEM_USE_MPI)
|
||||
find_file(
|
||||
CUDSS_COMM_LIB
|
||||
NAMES libcudss_commlayer_openmpi.so
|
||||
PATHS ${CUDSS_LIBRARY_DIR}
|
||||
NO_DEFAULT_PATH
|
||||
)
|
||||
if (NOT DEFINED MFEM_CUDSS_COMM_LIB AND CUDSS_COMM_LIB)
|
||||
set(MFEM_CUDSS_COMM_LIB "${CUDSS_COMM_LIB}")
|
||||
endif()
|
||||
message(STATUS "CUDSS communication layer library: ${MFEM_CUDSS_COMM_LIB}")
|
||||
endif()
|
||||
@@ -18,17 +18,19 @@
|
||||
|
||||
if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
|
||||
enable_language(C)
|
||||
set(GSLIB_FETCH_VERSION 1.0.9)
|
||||
add_library(GSLIB STATIC IMPORTED)
|
||||
# set options (technically flags because GSLIB does not use cmake)
|
||||
string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
|
||||
set(GSLIB_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
|
||||
if (BUILD_SHARED_LIBS)
|
||||
set(GSLIB_FLAGS "${GSLIB_FLAGS} -fPIC")
|
||||
set(GSLIB_FETCH_VERSION 1.0.9)
|
||||
set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}})
|
||||
if (CMAKE_C_FLAGS)
|
||||
set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
|
||||
endif()
|
||||
if (BUILD_SHARED_LIBS)
|
||||
set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC")
|
||||
endif()
|
||||
add_library(GSLIB STATIC IMPORTED)
|
||||
# define external project and create future include directory so it is present
|
||||
# to pass CMake checks at end of MFEM configuration step
|
||||
message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_FLAGS}")
|
||||
message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}")
|
||||
set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib)
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(gslib
|
||||
@@ -38,7 +40,7 @@ if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
|
||||
UPDATE_DISCONNECTED TRUE
|
||||
PREFIX ${PREFIX}
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS=${GSLIB_FLAGS}"
|
||||
BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS= ${GSLIB_C_FLAGS}"
|
||||
INSTALL_COMMAND "")
|
||||
file(MAKE_DIRECTORY ${PREFIX}/include)
|
||||
# set imported library target properties
|
||||
|
||||
@@ -44,9 +44,6 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
|
||||
# set options and associated dependencies
|
||||
set(HYPRE_CMAKE_OPTIONS "")
|
||||
list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
|
||||
if (BUILD_SHARED_LIBS)
|
||||
list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON)
|
||||
endif()
|
||||
# collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
|
||||
get_cmake_property(all_vars VARIABLES)
|
||||
foreach(var ${all_vars})
|
||||
@@ -98,6 +95,7 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
|
||||
UPDATE_DISCONNECTED TRUE
|
||||
SOURCE_SUBDIR src
|
||||
PREFIX ${HYPRE_INSTALL}
|
||||
BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL}
|
||||
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS})
|
||||
file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include)
|
||||
# set imported library target properties
|
||||
|
||||
@@ -19,18 +19,10 @@
|
||||
# - METIS_VERSION_5 (cache variable)
|
||||
|
||||
if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
|
||||
enable_language(C)
|
||||
set(METIS_FETCH_VERSION 4.0.3)
|
||||
add_library(METIS STATIC IMPORTED)
|
||||
# set options (technically flags because METIS does not use cmake)
|
||||
set(METIS_FLAGS "-Wno-implicit-int -Wno-incompatible-pointer-types")
|
||||
string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
|
||||
set(METIS_FLAGS "${METIS_FLAGS} ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
|
||||
if (BUILD_SHARED_LIBS)
|
||||
set(METIS_FLAGS "${METIS_FLAGS} -fPIC")
|
||||
endif()
|
||||
# define external project
|
||||
message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with ${METIS_FLAGS}")
|
||||
message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with default options")
|
||||
set(PREFIX ${CMAKE_BINARY_DIR}/fetch/metis)
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(metis
|
||||
@@ -40,7 +32,7 @@ if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
|
||||
UPDATE_DISCONNECTED TRUE
|
||||
PREFIX ${PREFIX}
|
||||
CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
|
||||
BUILD_COMMAND $(MAKE) clean && $(MAKE) "OPTFLAGS=${METIS_FLAGS}"
|
||||
BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types
|
||||
INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
|
||||
# set imported library target properties
|
||||
add_dependencies(METIS metis)
|
||||
|
||||
@@ -22,15 +22,15 @@ include(MfemCmakeUtilities)
|
||||
mfem_find_package(SuiteSparse SuiteSparse SuiteSparse_DIR "" "" "" ""
|
||||
"Paths to headers required by SuiteSparse."
|
||||
"Libraries required by SuiteSparse."
|
||||
ADD_COMPONENT "UMFPACK" "include;include/suitesparse;suitesparse" umfpack.h "lib" umfpack
|
||||
ADD_COMPONENT "KLU" "include;include/suitesparse;suitesparse" klu.h "lib" klu
|
||||
ADD_COMPONENT "AMD" "include;include/suitesparse;suitesparse" amd.h "lib" amd
|
||||
ADD_COMPONENT "BTF" "include;include/suitesparse;suitesparse" btf.h "lib" btf
|
||||
ADD_COMPONENT "CHOLMOD" "include;include/suitesparse;suitesparse" cholmod.h "lib" cholmod
|
||||
ADD_COMPONENT "COLAMD" "include;include/suitesparse;suitesparse" colamd.h "lib" colamd
|
||||
ADD_COMPONENT "CAMD" "include;include/suitesparse;suitesparse" camd.h "lib" camd
|
||||
ADD_COMPONENT "CCOLAMD" "include;include/suitesparse;suitesparse" ccolamd.h "lib" ccolamd
|
||||
ADD_COMPONENT "config" "include;include/suitesparse;suitesparse" SuiteSparse_config.h "lib"
|
||||
ADD_COMPONENT "UMFPACK" "include;suitesparse" umfpack.h "lib" umfpack
|
||||
ADD_COMPONENT "KLU" "include;suitesparse" klu.h "lib" klu
|
||||
ADD_COMPONENT "AMD" "include;suitesparse" amd.h "lib" amd
|
||||
ADD_COMPONENT "BTF" "include;suitesparse" btf.h "lib" btf
|
||||
ADD_COMPONENT "CHOLMOD" "include;suitesparse" cholmod.h "lib" cholmod
|
||||
ADD_COMPONENT "COLAMD" "include;suitesparse" colamd.h "lib" colamd
|
||||
ADD_COMPONENT "CAMD" "include;suitesparse" camd.h "lib" camd
|
||||
ADD_COMPONENT "CCOLAMD" "include;suitesparse" ccolamd.h "lib" ccolamd
|
||||
ADD_COMPONENT "config" "include;suitesparse" SuiteSparse_config.h "lib"
|
||||
suitesparseconfig)
|
||||
|
||||
if (SuiteSparse_FOUND AND METIS_VERSION_5)
|
||||
|
||||
+2
-8
@@ -18,9 +18,9 @@
|
||||
#define MFEM_CONFIG_HPP
|
||||
|
||||
#ifdef MFEM_CONFIG_FILE
|
||||
#include MFEM_CONFIG_FILE // IWYU pragma: export
|
||||
#include MFEM_CONFIG_FILE
|
||||
#else
|
||||
#include "_config.hpp" // IWYU pragma: export
|
||||
#include "_config.hpp"
|
||||
#endif
|
||||
|
||||
#include <cstdint>
|
||||
@@ -157,10 +157,4 @@ constexpr real_t operator""_r(unsigned long long v)
|
||||
#endif
|
||||
#endif // MFEM_USE_MPI not defined
|
||||
|
||||
#ifndef MFEM_USE_CUDA
|
||||
#ifdef MFEM_USE_CUDSS
|
||||
#error Building with cuDSS (MFEM_USE_CUDSS=YES) requires CUDA (MFEM_USE_CUDA=YES)
|
||||
#endif
|
||||
#endif // MFEM_USE_CUDSS not defined
|
||||
|
||||
#endif // MFEM_CONFIG_HPP
|
||||
|
||||
@@ -97,6 +97,9 @@
|
||||
// Enable MFEM functionality based on the SuiteSparse library.
|
||||
// #define MFEM_USE_SUITESPARSE
|
||||
|
||||
// Enable MFEM functionality based on the ARPACK library.
|
||||
// #define MFEM_USE_ARPACK
|
||||
|
||||
// Enable MFEM functionality based on the SuperLU_DIST library.
|
||||
// #define MFEM_USE_SUPERLU
|
||||
// #define MFEM_USE_SUPERLU5
|
||||
@@ -108,15 +111,6 @@
|
||||
// Enable MFEM functionality based on the STRUMPACK library.
|
||||
// #define MFEM_USE_STRUMPACK
|
||||
|
||||
// Enable MFEM functionality based on the cuDSS library.
|
||||
// #define MFEM_USE_CUDSS
|
||||
|
||||
// CUDSS communication layer library path
|
||||
// #define MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
|
||||
|
||||
// CUDSS threading layer library path
|
||||
// #define MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
|
||||
|
||||
// Enable MFEM features based on the Ginkgo library.
|
||||
// #define MFEM_USE_GINKGO
|
||||
|
||||
|
||||
+1
-3
@@ -32,13 +32,11 @@ MFEM_USE_MEMALLOC = @MFEM_USE_MEMALLOC@
|
||||
MFEM_TIMER_TYPE = @MFEM_TIMER_TYPE@
|
||||
MFEM_USE_SUNDIALS = @MFEM_USE_SUNDIALS@
|
||||
MFEM_USE_SUITESPARSE = @MFEM_USE_SUITESPARSE@
|
||||
MFEM_USE_ARPACK = @MFEM_USE_ARPACK@
|
||||
MFEM_USE_SUPERLU = @MFEM_USE_SUPERLU@
|
||||
MFEM_USE_SUPERLU5 = @MFEM_USE_SUPERLU5@
|
||||
MFEM_USE_MUMPS = @MFEM_USE_MUMPS@
|
||||
MFEM_USE_STRUMPACK = @MFEM_USE_STRUMPACK@
|
||||
MFEM_USE_CUDSS = @MFEM_USE_CUDSS@
|
||||
MFEM_CUDSS_COMM_LIB = @MFEM_CUDSS_COMM_LIB@
|
||||
MFEM_CUDSS_THREADING_LIB = @MFEM_CUDSS_THREADING_LIB@
|
||||
MFEM_USE_GINKGO = @MFEM_USE_GINKGO@
|
||||
MFEM_USE_AMGX = @MFEM_USE_AMGX@
|
||||
MFEM_USE_MAGMA = @MFEM_USE_MAGMA@
|
||||
|
||||
@@ -38,7 +38,6 @@ option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
|
||||
option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
|
||||
option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
|
||||
option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
|
||||
option(MFEM_USE_CUDSS "Enable cuDSS usage" OFF)
|
||||
option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
|
||||
option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
|
||||
option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
|
||||
|
||||
+11
-22
@@ -27,13 +27,7 @@ MPICXX = mpicxx
|
||||
|
||||
BASE_FLAGS = -std=c++17
|
||||
OPTIM_FLAGS = -O3 $(BASE_FLAGS)
|
||||
|
||||
# Shadow warnings for clang only; GCC's -Wshadow flags more.
|
||||
SHADOW_WARNING_FLAG = $(if $(findstring clang,\
|
||||
$(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
|
||||
WARNING_FLAGS = -pedantic -Wall $(SHADOW_WARNING_FLAG)
|
||||
|
||||
DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
|
||||
DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
|
||||
|
||||
# Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
|
||||
CXX_XCOMPILER =
|
||||
@@ -159,7 +153,6 @@ MFEM_USE_SUPERLU = NO
|
||||
MFEM_USE_SUPERLU5 = NO
|
||||
MFEM_USE_MUMPS = NO
|
||||
MFEM_USE_STRUMPACK = NO
|
||||
MFEM_USE_CUDSS = NO
|
||||
MFEM_USE_GINKGO = NO
|
||||
MFEM_USE_AMGX = NO
|
||||
MFEM_USE_MAGMA = NO
|
||||
@@ -185,6 +178,7 @@ MFEM_USE_ALGOIM = NO
|
||||
MFEM_USE_UMPIRE = NO
|
||||
MFEM_USE_SIMD = NO
|
||||
MFEM_USE_ADIOS2 = NO
|
||||
MFEM_USE_ARPACK = NO
|
||||
MFEM_USE_MKL_CPARDISO = NO
|
||||
MFEM_USE_MKL_PARDISO = NO
|
||||
MFEM_USE_MOONOLITH = NO
|
||||
@@ -375,19 +369,6 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
|
||||
STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
|
||||
$(SCOTCH_LIB) $(SCALAPACK_LIB)
|
||||
|
||||
# CUDSS library configuration
|
||||
CUDSS_DIR = @MFEM_DIR@/../cudss
|
||||
CUDSS_INCLUDE_DIR = $(CUDSS_DIR)/include
|
||||
CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
|
||||
CUDSS_OPT = -I$(CUDSS_INCLUDE_DIR)
|
||||
CUDSS_LIB = \
|
||||
$(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
|
||||
# The cuDSS communication and threading libraries.
|
||||
MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
|
||||
$(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
|
||||
MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
|
||||
$(subst @MFEM_DIR@,$(MFEM_DIR),$(CUDSS_LIBRARY_DIR)/libcudss_mtlayer_gomp.so))))
|
||||
|
||||
# Ginkgo library configuration
|
||||
GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
|
||||
GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
|
||||
@@ -447,6 +428,14 @@ NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
|
||||
$(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
|
||||
-lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)
|
||||
|
||||
# ARPACK library configuration
|
||||
ARPACK_DIR = @MFEM_DIR@/../ARPACK
|
||||
ifeq ($(MFEM_USE_MPI),YES)
|
||||
ARPACK_LIB = -L$(ARPACK_DIR) -lparpack -larpack
|
||||
else
|
||||
ARPACK_LIB = -L$(ARPACK_DIR) -larpack
|
||||
endif
|
||||
|
||||
# PETSc library configuration (version greater or equal to 3.8 or the dev branch)
|
||||
PETSC_ARCH := arch-linux2-c-debug
|
||||
PETSC_DIR := $(MFEM_DIR)/../petsc/$(PETSC_ARCH)
|
||||
@@ -641,7 +630,7 @@ PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
|
||||
AXOM_DIR = @MFEM_DIR@/../axom
|
||||
TRIBOL_DIR = @MFEM_DIR@/../tribol
|
||||
TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
|
||||
TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -ltribol_shared -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
|
||||
TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
|
||||
-laxom_slam -laxom_slic -laxom_core
|
||||
|
||||
# Enzyme configuration
|
||||
|
||||
@@ -39,8 +39,3 @@ when a picture was added for documentation.
|
||||
If that is the case, make sure the failure is indeed justified, and rerun the
|
||||
push command with the `--no-verify` option. This will skip the hooks, allowing
|
||||
you to push those changes.
|
||||
|
||||
The `branch-history` check is run automatically through GitHub Actions. If a
|
||||
branch is known to have a large number of changes that are legitimate, the
|
||||
check can be overridden by setting the label 'branch-history-override' on the
|
||||
pull request.
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
MFEM mesh v1.0
|
||||
|
||||
#
|
||||
# MFEM Geometry Types (see fem/geom.hpp):
|
||||
#
|
||||
# POINT = 0
|
||||
# SEGMENT = 1
|
||||
# TRIANGLE = 2
|
||||
# SQUARE = 3
|
||||
# TETRAHEDRON = 4
|
||||
# CUBE = 5
|
||||
# PRISM = 6
|
||||
# PYRAMID = 7
|
||||
|
||||
dimension
|
||||
2
|
||||
|
||||
elements
|
||||
2
|
||||
1 3 0 1 4 3
|
||||
1 2 1 2 4
|
||||
|
||||
boundary
|
||||
5
|
||||
1 1 0 1
|
||||
1 1 1 2
|
||||
1 1 2 4
|
||||
1 1 4 3
|
||||
1 1 3 0
|
||||
|
||||
vertices
|
||||
5
|
||||
2
|
||||
0 0
|
||||
1 0
|
||||
2 0
|
||||
0 1
|
||||
1 1
|
||||
@@ -201,7 +201,6 @@ namespace mfem {
|
||||
* - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
|
||||
* - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
|
||||
* - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
|
||||
* - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
|
||||
*
|
||||
* <H3>Miniapps</H3>
|
||||
* - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
|
||||
@@ -246,9 +245,6 @@ namespace mfem {
|
||||
* - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
|
||||
* - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
|
||||
* - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
|
||||
* - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
|
||||
* - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
|
||||
* - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
|
||||
*
|
||||
* See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
|
||||
*/
|
||||
|
||||
@@ -47,9 +47,15 @@ list(APPEND ALL_EXE_SRCS
|
||||
ex39.cpp
|
||||
ex40.cpp
|
||||
ex41.cpp
|
||||
# jitplayground.cpp
|
||||
)
|
||||
|
||||
if (MFEM_USE_ARPACK)
|
||||
list(APPEND ALL_EXE_SRCS
|
||||
ex11.pp
|
||||
ex13.pp
|
||||
)
|
||||
endif()
|
||||
|
||||
if (MFEM_USE_MPI)
|
||||
list(APPEND ALL_EXE_SRCS
|
||||
ex0p.cpp
|
||||
@@ -216,7 +222,7 @@ if (MFEM_ENABLE_TESTING)
|
||||
add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
|
||||
COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
|
||||
${MPIEXEC_PREFLAGS}
|
||||
$<TARGET_FILE:ex1p> "-no-vis" "-d" "ceed-cpu" "-pa" "-a"
|
||||
$<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
|
||||
${MPIEXEC_POSTFLAGS})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -64,7 +64,7 @@ PARALLEL_NAME := Parallel AMGX example
|
||||
$(MFEM_LIB_FILE):
|
||||
$(error The MFEM library is not build)
|
||||
|
||||
clean: clean-build clean-exec
|
||||
clean: clean-build
|
||||
|
||||
clean-build:
|
||||
rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
|
||||
|
||||
@@ -64,12 +64,12 @@ ex1p-test-par: ex1p
|
||||
$(MFEM_LIB_FILE):
|
||||
$(error The MFEM library is not built)
|
||||
|
||||
clean: clean-build clean-exec
|
||||
clean: clean-build clean-exec $(SUBDIRS_CLEAN)
|
||||
|
||||
clean-build:
|
||||
rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
|
||||
rm -rf *.dSYM *.TVD.*breakpoints
|
||||
|
||||
clean-exec:
|
||||
@rm -f refined.mesh mesh.*
|
||||
@rm -f sol.*
|
||||
@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
|
||||
@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
|
||||
|
||||
+21
-34
@@ -50,10 +50,6 @@
|
||||
// ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
|
||||
// ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
|
||||
//
|
||||
// Device simplices sample runs:
|
||||
// ex1 -pa -d gpu -m ../data/inline-tet.mesh
|
||||
// ex1 -pa -d gpu -m ../data/inline-tri.mesh
|
||||
//
|
||||
// Description: This example code demonstrates the use of MFEM to define a
|
||||
// simple finite element discretization of the Poisson problem
|
||||
// -Delta u = 1 with homogeneous Dirichlet boundary conditions.
|
||||
@@ -142,25 +138,25 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
// 5. Define a finite element space on the mesh. Here we use continuous
|
||||
// Lagrange finite elements of the specified order.
|
||||
// - If order < 1, we instead use an isoparametric/isogeometric space.
|
||||
// - If the mesh is simplicial and partial assembly is requested,
|
||||
// we use the positive basis, which supports device execution.
|
||||
// Lagrange finite elements of the specified order. If order < 1, we
|
||||
// instead use an isoparametric/isogeometric space.
|
||||
FiniteElementCollection *fec;
|
||||
auto basis_type = (pa && mesh.IsSimplexMesh()) ?
|
||||
BasisType::Positive : BasisType::GaussLobatto;
|
||||
bool delete_fec;
|
||||
if (order > 0)
|
||||
{
|
||||
fec = new H1_FECollection(order, dim, basis_type);
|
||||
fec = new H1_FECollection(order, dim);
|
||||
delete_fec = true;
|
||||
}
|
||||
else if (mesh.GetNodes())
|
||||
{
|
||||
fec = mesh.GetNodes()->OwnFEC();
|
||||
delete_fec = false;
|
||||
cout << "Using isoparametric FEs: " << fec->Name() << endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
fec = new H1_FECollection(order = 1, dim, basis_type);
|
||||
fec = new H1_FECollection(order = 1, dim);
|
||||
delete_fec = true;
|
||||
}
|
||||
FiniteElementSpace fespace(&mesh, fec);
|
||||
cout << "Number of finite element unknowns: "
|
||||
@@ -228,29 +224,17 @@ int main(int argc, char *argv[])
|
||||
// 11. Solve the linear system A X = B.
|
||||
if (!pa)
|
||||
{
|
||||
#ifdef MFEM_USE_CUDSS
|
||||
if (Device::Allows(Backend::CUDA_MASK))
|
||||
{
|
||||
// Use cuDSS to solve the system.
|
||||
CuDSSSolver cudss_solver;
|
||||
cudss_solver.SetOperator(*A);
|
||||
cudss_solver.Mult(B, X);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
#ifndef MFEM_USE_SUITESPARSE
|
||||
// Use a simple symmetric Gauss-Seidel preconditioner with PCG.
|
||||
GSSmoother M((SparseMatrix&)(*A));
|
||||
PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
|
||||
// Use a simple symmetric Gauss-Seidel preconditioner with PCG.
|
||||
GSSmoother M((SparseMatrix&)(*A));
|
||||
PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
|
||||
#else
|
||||
// If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
|
||||
UMFPackSolver umf_solver;
|
||||
umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
|
||||
umf_solver.SetOperator(*A);
|
||||
umf_solver.Mult(B, X);
|
||||
// If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
|
||||
UMFPackSolver umf_solver;
|
||||
umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
|
||||
umf_solver.SetOperator(*A);
|
||||
umf_solver.Mult(B, X);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -289,14 +273,17 @@ int main(int argc, char *argv[])
|
||||
if (visualization)
|
||||
{
|
||||
char vishost[] = "localhost";
|
||||
int visport = 19916;
|
||||
int visport = 19916;
|
||||
socketstream sol_sock(vishost, visport);
|
||||
sol_sock.precision(8);
|
||||
sol_sock << "solution\n" << mesh << x << flush;
|
||||
}
|
||||
|
||||
// 15. Free the used memory.
|
||||
if (order > 0) { delete fec; }
|
||||
if (delete_fec)
|
||||
{
|
||||
delete fec;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
// MFEM Example 11 - Serial Version
|
||||
//
|
||||
// Compile with: make ex11
|
||||
//
|
||||
// Sample runs: ex11 -m ../data/square-disc.mesh
|
||||
// ex11 -m ../data/star.mesh
|
||||
// ex11 -m ../data/star-mixed.mesh
|
||||
// ex11 -m ../data/periodic-annulus-sector.msh
|
||||
// ex11 -m ../data/square-disc-p2.vtk -o 2
|
||||
// ex11 -m ../data/square-disc-p3.mesh -o 3
|
||||
// ex11 -m ../data/square-disc-nurbs.mesh -o -1
|
||||
// ex11 -m ../data/disc-nurbs.mesh -o -1 -n 20
|
||||
// ex11 -m ../data/star-surf.mesh
|
||||
// ex11 -m ../data/square-disc-surf.mesh
|
||||
// ex11 -m ../data/inline-segment.mesh
|
||||
// ex11 -m ../data/inline-quad.mesh
|
||||
// ex11 -m ../data/inline-tri.mesh
|
||||
// ex11 -m ../data/amr-quad.mesh
|
||||
// ex11 -m ../data/amr-hex.mesh
|
||||
// ex11 -m ../data/mobius-strip.mesh -n 8
|
||||
//
|
||||
// Description: This example code demonstrates the use of MFEM to solve the
|
||||
// eigenvalue problem -Delta u = lambda u with homogeneous
|
||||
// Dirichlet boundary conditions.
|
||||
//
|
||||
// We compute a number of the lowest eigenmodes by discretizing
|
||||
// the Laplacian and Mass operators using a FE space of the
|
||||
// specified order, or an isoparametric/isogeometric space if
|
||||
// order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
|
||||
// NURBS mesh, etc.)
|
||||
//
|
||||
// The example highlights the use of the ARPACK eigenvalue solver
|
||||
// (regular inverse mode). Reusing a single GLVis visualization
|
||||
// window for multiple eigenfunctions is also illustrated.
|
||||
//
|
||||
// We recommend viewing Example 1 before viewing this example.
|
||||
|
||||
#include "mfem.hpp"
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
|
||||
using namespace std;
|
||||
using namespace mfem;
|
||||
|
||||
#ifdef MFEM_USE_ARPACK
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
// 1. Parse command-line options.
|
||||
const char *mesh_file = "../data/star.mesh";
|
||||
int ser_ref_levels = 3;
|
||||
int order = 1;
|
||||
int nev = 5;
|
||||
double dbc_eig = 1e3;
|
||||
bool visualization = 1;
|
||||
bool arp_solver = true;
|
||||
|
||||
OptionsParser args(argc, argv);
|
||||
args.AddOption(&mesh_file, "-m", "--mesh",
|
||||
"Mesh file to use.");
|
||||
args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
|
||||
"Number of times to refine the mesh uniformly in serial.");
|
||||
args.AddOption(&order, "-o", "--order",
|
||||
"Finite element order (polynomial degree) or -1 for"
|
||||
" isoparametric space.");
|
||||
args.AddOption(&nev, "-n", "--num-eigs",
|
||||
"Number of desired eigenmodes.");
|
||||
args.AddOption(&dbc_eig, "-d", "--dbc-eig",
|
||||
"Eigenvalues associated with Dirichlet BC "
|
||||
"(should be larger than the maximum desired eigenvalue).");
|
||||
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
|
||||
"--no-visualization",
|
||||
"Enable or disable GLVis visualization.");
|
||||
args.Parse();
|
||||
if (!args.Good())
|
||||
{
|
||||
args.PrintUsage(cout);
|
||||
return 1;
|
||||
}
|
||||
args.PrintOptions(cout);
|
||||
|
||||
// 2. Read the (serial) mesh from the given mesh file on all processors. We
|
||||
// can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
|
||||
// and volume meshes with the same code.
|
||||
Mesh *mesh;
|
||||
ifstream imesh(mesh_file);
|
||||
if (!imesh)
|
||||
{
|
||||
cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
|
||||
return 2;
|
||||
}
|
||||
mesh = new Mesh(imesh, 1, 1);
|
||||
imesh.close();
|
||||
int dim = mesh->Dimension();
|
||||
|
||||
// 3. Refine the serial mesh on all processors to increase the resolution. In
|
||||
// this example we do 'ref_levels' of uniform refinement (2 by default, or
|
||||
// specified on the command line with -rs).
|
||||
for (int lev = 0; lev < ser_ref_levels; lev++)
|
||||
{
|
||||
mesh->UniformRefinement();
|
||||
}
|
||||
|
||||
// 4. Define a finite element space on the mesh. Here we
|
||||
// use continuous Lagrange finite elements of the specified order. If
|
||||
// order < 1, we instead use an isoparametric/isogeometric space.
|
||||
FiniteElementCollection *fec;
|
||||
if (order > 0)
|
||||
{
|
||||
fec = new H1_FECollection(order, dim);
|
||||
}
|
||||
else if (mesh->GetNodes())
|
||||
{
|
||||
fec = mesh->GetNodes()->OwnFEC();
|
||||
}
|
||||
else
|
||||
{
|
||||
fec = new H1_FECollection(order = 1, dim);
|
||||
}
|
||||
FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
|
||||
int size = fespace->GetVSize();
|
||||
|
||||
cout << "Number of unknowns: " << size << endl;
|
||||
|
||||
// 5. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
|
||||
// element space. The first corresponds to the Laplacian operator -Delta,
|
||||
// while the second is a simple mass matrix needed on the right hand side
|
||||
// of the generalized eigenvalue problem below. The boundary conditions
|
||||
// are implemented by elimination with special values on the diagonal to
|
||||
// shift the Dirichlet eigenvalues out of the computational range. After
|
||||
// serial and parallel assembly we extract the corresponding parallel
|
||||
// matrices A and M.
|
||||
ConstantCoefficient one(1.0);
|
||||
Array<int> ess_bdr;
|
||||
if (mesh->bdr_attributes.Size())
|
||||
{
|
||||
ess_bdr.SetSize(mesh->bdr_attributes.Max());
|
||||
ess_bdr = 1;
|
||||
}
|
||||
|
||||
BilinearForm *a = new BilinearForm(fespace);
|
||||
a->AddDomainIntegrator(new DiffusionIntegrator(one));
|
||||
if (mesh->bdr_attributes.Size() == 0)
|
||||
{
|
||||
// Add a mass term if the mesh has no boundary, e.g. periodic mesh or
|
||||
// closed surface.
|
||||
a->AddDomainIntegrator(new MassIntegrator(one));
|
||||
}
|
||||
a->Assemble();
|
||||
if (mesh->bdr_attributes.Size() != 0)
|
||||
{
|
||||
a->EliminateEssentialBCDiag(ess_bdr, dbc_eig);
|
||||
}
|
||||
a->Finalize();
|
||||
|
||||
BilinearForm *m = new BilinearForm(fespace);
|
||||
m->AddDomainIntegrator(new MassIntegrator(one));
|
||||
m->Assemble();
|
||||
if (mesh->bdr_attributes.Size() != 0)
|
||||
{
|
||||
// shift the eigenvalue corresponding to eliminated dofs to a large value
|
||||
m->EliminateEssentialBCDiag(ess_bdr, 1.0);
|
||||
}
|
||||
m->Finalize();
|
||||
|
||||
Solver * solver = NULL;
|
||||
|
||||
#ifndef MFEM_USE_SUITESPARSE
|
||||
// 6. Define a simple symmetric Gauss-Seidel preconditioner and use it to
|
||||
// solve the system A X = B with PCG.
|
||||
cout << "Building CGSolver" << endl;
|
||||
GSSmoother M(m->SpMat());
|
||||
CGSolver * cg_solver = new CGSolver;
|
||||
cg_solver->SetPreconditioner(M);
|
||||
cg_solver->SetRelTol(1.0e-12);
|
||||
solver = cg_solver;
|
||||
#else
|
||||
// 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
|
||||
cout << "Building UMFPackSolver" << endl;
|
||||
UMFPackSolver * umf_solver = new UMFPackSolver;
|
||||
umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
|
||||
solver = umf_solver;
|
||||
#endif
|
||||
solver->SetOperator(m->SpMat());
|
||||
|
||||
// 7. Define and configure the ARPACK eigensolver
|
||||
SymGenEigensolver * eig_solver = NULL;
|
||||
|
||||
if (arp_solver)
|
||||
{
|
||||
// ArPackSymGen * arpack = new ArPackSymGen();
|
||||
ArPackSAUPD * arpack = new ArPackSAUPD();
|
||||
arpack->SetMode(2);
|
||||
arpack->SetNumModes(nev);
|
||||
arpack->SetMaxIter(400);
|
||||
arpack->SetTol(1e-8);
|
||||
arpack->SetPrintLevel(2);
|
||||
arpack->SetSolver(*solver);
|
||||
|
||||
eig_solver = arpack;
|
||||
}
|
||||
|
||||
eig_solver->SetOperators(*a, *m);
|
||||
|
||||
// 8. Compute the eigenmodes and extract the array of eigenvalues. Define a
|
||||
// parallel grid function to represent each of the eigenmodes returned by
|
||||
// the solver.
|
||||
Array<double> eigenvalues;
|
||||
eig_solver->Solve();
|
||||
eig_solver->GetEigenvalues(eigenvalues);
|
||||
|
||||
cout << endl;
|
||||
std::ios::fmtflags old_fmt = cout.flags();
|
||||
cout.setf(std::ios::scientific);
|
||||
std::streamsize old_prec = cout.precision(14);
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
cout << "Eigenvalue lambda " << eigenvalues[i] << endl;
|
||||
}
|
||||
cout.precision(old_prec);
|
||||
cout.flags(old_fmt);
|
||||
cout << endl;
|
||||
|
||||
GridFunction x(fespace);
|
||||
|
||||
// 9. Save the refined mesh and the modes in parallel. This output can be
|
||||
// viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
|
||||
{
|
||||
ostringstream mesh_name, mode_name;
|
||||
mesh_name << "ex11.mesh";
|
||||
|
||||
ofstream mesh_ofs(mesh_name.str().c_str());
|
||||
mesh_ofs.precision(8);
|
||||
mesh->Print(mesh_ofs);
|
||||
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
// convert eigenvector from Vector to GridFunction
|
||||
x = eig_solver->GetEigenvector(i);
|
||||
|
||||
mode_name << "mode_" << setfill('0') << setw(2) << i;
|
||||
|
||||
ofstream mode_ofs(mode_name.str().c_str());
|
||||
mode_ofs.precision(8);
|
||||
x.Save(mode_ofs);
|
||||
mode_name.str("");
|
||||
}
|
||||
}
|
||||
|
||||
// 10. Send the solution by socket to a GLVis server.
|
||||
if (visualization)
|
||||
{
|
||||
char vishost[] = "localhost";
|
||||
int visport = 19916;
|
||||
socketstream mode_sock(vishost, visport);
|
||||
mode_sock.precision(8);
|
||||
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
cout << "Eigenmode " << i+1 << '/' << nev
|
||||
<< ", Lambda = " << eigenvalues[i] << endl;
|
||||
|
||||
// convert eigenvector from Vector to GridFunction
|
||||
x = eig_solver->GetEigenvector(i);
|
||||
|
||||
mode_sock << "solution\n" << *mesh << x << flush
|
||||
<< "window_title 'Eigenmode " << i+1 << '/' << nev
|
||||
<< ", Lambda = " << eigenvalues[i] << "'" << endl;
|
||||
|
||||
char c;
|
||||
cout << "press (q)uit or (c)ontinue --> " << flush;
|
||||
cin >> c;
|
||||
|
||||
if (c != 'c')
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
mode_sock.close();
|
||||
}
|
||||
|
||||
// 11. Free the used memory.
|
||||
delete eig_solver;
|
||||
delete solver;
|
||||
delete m;
|
||||
delete a;
|
||||
|
||||
delete fespace;
|
||||
if (order > 0)
|
||||
{
|
||||
delete fec;
|
||||
}
|
||||
delete mesh;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // MFEM_USE_ARPACK
|
||||
+104
-43
@@ -72,6 +72,8 @@ int main(int argc, char *argv[])
|
||||
int seed = 75;
|
||||
bool slu_solver = false;
|
||||
bool sp_solver = false;
|
||||
bool lob_solver = true;
|
||||
bool arp_solver = false;
|
||||
bool cpardiso_solver = false;
|
||||
bool visualization = 1;
|
||||
|
||||
@@ -97,6 +99,10 @@ int main(int argc, char *argv[])
|
||||
args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
|
||||
"--no-strumpack", "Use the STRUMPACK Solver.");
|
||||
#endif
|
||||
#ifdef MFEM_USE_ARPACK
|
||||
args.AddOption(&arp_solver, "-arp", "--arpack", "-no-arp",
|
||||
"--no-arpack", "Use the Parallel ARPACK Solver.");
|
||||
#endif
|
||||
#ifdef MFEM_USE_MKL_CPARDISO
|
||||
args.AddOption(&cpardiso_solver, "-cpardiso", "--cpardiso", "-no-cpardiso",
|
||||
"--no-cpardiso", "Use the MKL CPardiso Solver.");
|
||||
@@ -113,6 +119,11 @@ int main(int argc, char *argv[])
|
||||
<< " Defaulting to SuperLU." << endl;
|
||||
sp_solver = false;
|
||||
}
|
||||
if (arp_solver)
|
||||
{
|
||||
lob_solver = false;
|
||||
}
|
||||
|
||||
// The command line options are also passed to the STRUMPACK
|
||||
// solver. So do not exit if some options are not recognized.
|
||||
if (!sp_solver)
|
||||
@@ -243,70 +254,119 @@ int main(int argc, char *argv[])
|
||||
// 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
|
||||
// preconditioner for A to be used within the solver. Set the matrices
|
||||
// which define the generalized eigenproblem A x = lambda M x.
|
||||
Solver * solver = NULL;
|
||||
Solver * precond = NULL;
|
||||
if (!slu_solver && !sp_solver && !cpardiso_solver)
|
||||
{
|
||||
HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
|
||||
amg->SetPrintLevel(0);
|
||||
precond = amg;
|
||||
}
|
||||
else
|
||||
{
|
||||
#ifdef MFEM_USE_SUPERLU
|
||||
if (slu_solver)
|
||||
|
||||
if (arp_solver)
|
||||
{
|
||||
HyprePCG * pcg = new HyprePCG(*A);
|
||||
pcg->SetTol(1e-12);
|
||||
pcg->SetPreconditioner(*amg);
|
||||
solver = pcg;
|
||||
}
|
||||
}
|
||||
#ifdef MFEM_USE_SUPERLU
|
||||
else if (slu_solver)
|
||||
{
|
||||
SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
|
||||
superlu->SetPrintStatistics(false);
|
||||
superlu->SetSymmetricPattern(true);
|
||||
superlu->SetColumnPermutation(superlu::PARMETIS);
|
||||
superlu->SetOperator(*Arow);
|
||||
|
||||
if (arp_solver)
|
||||
{
|
||||
solver = superlu;
|
||||
}
|
||||
else
|
||||
{
|
||||
SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
|
||||
superlu->SetPrintStatistics(false);
|
||||
superlu->SetSymmetricPattern(true);
|
||||
superlu->SetColumnPermutation(superlu::PARMETIS);
|
||||
superlu->SetOperator(*Arow);
|
||||
precond = superlu;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#ifdef MFEM_USE_STRUMPACK
|
||||
if (sp_solver)
|
||||
else if (sp_solver)
|
||||
{
|
||||
STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv,
|
||||
MPI_COMM_WORLD);
|
||||
strumpack->SetPrintFactorStatistics(true);
|
||||
strumpack->SetPrintSolveStatistics(false);
|
||||
strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
|
||||
strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
|
||||
strumpack->SetMatching(strumpack::MatchingJob::NONE);
|
||||
strumpack->SetCompression(strumpack::CompressionType::NONE);
|
||||
strumpack->SetOperator(*Arow);
|
||||
strumpack->SetFromCommandLine();
|
||||
if (arp_solver)
|
||||
{
|
||||
solver = strumpack;
|
||||
}
|
||||
else
|
||||
{
|
||||
STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
|
||||
strumpack->SetPrintFactorStatistics(true);
|
||||
strumpack->SetPrintSolveStatistics(false);
|
||||
strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
|
||||
strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
|
||||
strumpack->SetMatching(strumpack::MatchingJob::NONE);
|
||||
strumpack->SetCompression(strumpack::CompressionType::NONE);
|
||||
strumpack->SetOperator(*Arow);
|
||||
strumpack->SetFromCommandLine();
|
||||
precond = strumpack;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#ifdef MFEM_USE_MKL_CPARDISO
|
||||
if (cpardiso_solver)
|
||||
else if (cpardiso_solver)
|
||||
{
|
||||
auto cpardiso = new CPardisoSolver(A->GetComm());
|
||||
cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
|
||||
cpardiso->SetPrintLevel(1);
|
||||
cpardiso->SetOperator(*A);
|
||||
if (arp_solver)
|
||||
{
|
||||
solver = cpardiso;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto cpardiso = new CPardisoSolver(A->GetComm());
|
||||
cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
|
||||
cpardiso->SetPrintLevel(1);
|
||||
cpardiso->SetOperator(*A);
|
||||
precond = cpardiso;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
|
||||
lobpcg->SetNumModes(nev);
|
||||
lobpcg->SetRandomSeed(seed);
|
||||
lobpcg->SetPreconditioner(*precond);
|
||||
lobpcg->SetMaxIter(200);
|
||||
lobpcg->SetTol(1e-8);
|
||||
lobpcg->SetPrecondUsageMode(1);
|
||||
lobpcg->SetPrintLevel(1);
|
||||
lobpcg->SetMassMatrix(*M);
|
||||
lobpcg->SetOperator(*A);
|
||||
SymGenEigensolver * eig_solver = NULL;
|
||||
|
||||
if (lob_solver)
|
||||
{
|
||||
HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
|
||||
lobpcg->SetNumModes(nev);
|
||||
lobpcg->SetRandomSeed(seed);
|
||||
lobpcg->SetPreconditioner(*precond);
|
||||
lobpcg->SetMaxIter(200);
|
||||
lobpcg->SetTol(1e-8);
|
||||
lobpcg->SetPrecondUsageMode(1);
|
||||
lobpcg->SetPrintLevel(1);
|
||||
|
||||
eig_solver = lobpcg;
|
||||
}
|
||||
#ifdef MFEM_USE_ARPACK
|
||||
else if (arp_solver)
|
||||
{
|
||||
ArPackPSAUPD * arpack = new ArPackPSAUPD(MPI_COMM_WORLD);
|
||||
arpack->SetNumModes(nev);
|
||||
arpack->SetMaxIter(400);
|
||||
arpack->SetTol(1e-8);
|
||||
arpack->SetMode(3);
|
||||
arpack->SetPrintLevel(2);
|
||||
arpack->SetSolver(*solver);
|
||||
|
||||
eig_solver = arpack;
|
||||
}
|
||||
#endif
|
||||
eig_solver->SetOperators(*A, *M);
|
||||
|
||||
// 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
|
||||
// parallel grid function to represent each of the eigenmodes returned by
|
||||
// the solver.
|
||||
Array<real_t> eigenvalues;
|
||||
lobpcg->Solve();
|
||||
lobpcg->GetEigenvalues(eigenvalues);
|
||||
eig_solver->Solve();
|
||||
eig_solver->GetEigenvalues(eigenvalues);
|
||||
ParGridFunction x(fespace);
|
||||
|
||||
// 10. Save the refined mesh and the modes in parallel. This output can be
|
||||
@@ -321,8 +381,8 @@ int main(int argc, char *argv[])
|
||||
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = lobpcg->GetEigenvector(i);
|
||||
// convert eigenvector from Vector to ParGridFunction
|
||||
x.Distribute(eig_solver->GetEigenvector(i));
|
||||
|
||||
mode_name << "mode_" << setfill('0') << setw(2) << i << "."
|
||||
<< setfill('0') << setw(6) << myid;
|
||||
@@ -350,8 +410,8 @@ int main(int argc, char *argv[])
|
||||
<< ", Lambda = " << eigenvalues[i] << endl;
|
||||
}
|
||||
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = lobpcg->GetEigenvector(i);
|
||||
// convert eigenvector from Vector to ParGridFunction
|
||||
x.Distribute(eig_solver->GetEigenvector(i));
|
||||
|
||||
mode_sock << "parallel " << num_procs << " " << myid << "\n"
|
||||
<< "solution\n" << *pmesh << x << flush
|
||||
@@ -375,7 +435,8 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
// 12. Free the used memory.
|
||||
delete lobpcg;
|
||||
delete eig_solver;
|
||||
delete solver;
|
||||
delete precond;
|
||||
delete M;
|
||||
delete A;
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
// MFEM Example 11 - Parallel Version
|
||||
//
|
||||
// Compile with: make ex11p
|
||||
//
|
||||
// Sample runs: mpirun -np 4 ex11p -m ../data/square-disc.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/star.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/escher.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/fichera.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/square-disc-p2.vtk -o 2
|
||||
// mpirun -np 4 ex11p -m ../data/square-disc-p3.mesh -o 3
|
||||
// mpirun -np 4 ex11p -m ../data/square-disc-nurbs.mesh -o -1
|
||||
// mpirun -np 4 ex11p -m ../data/disc-nurbs.mesh -o -1 -n 20
|
||||
// mpirun -np 4 ex11p -m ../data/pipe-nurbs.mesh -o -1
|
||||
// mpirun -np 4 ex11p -m ../data/ball-nurbs.mesh -o 2
|
||||
// mpirun -np 4 ex11p -m ../data/star-surf.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/square-disc-surf.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/inline-segment.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/amr-quad.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/amr-hex.mesh
|
||||
// mpirun -np 4 ex11p -m ../data/mobius-strip.mesh -n 8
|
||||
// mpirun -np 4 ex11p -m ../data/klein-bottle.mesh -n 10
|
||||
//
|
||||
// Description: This example code demonstrates the use of MFEM to solve the
|
||||
// eigenvalue problem -Delta u = lambda u with homogeneous
|
||||
// Dirichlet boundary conditions.
|
||||
//
|
||||
// We compute a number of the lowest eigenmodes by discretizing
|
||||
// the Laplacian and Mass operators using a FE space of the
|
||||
// specified order, or an isoparametric/isogeometric space if
|
||||
// order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
|
||||
// NURBS mesh, etc.)
|
||||
//
|
||||
// The example highlights the use of the LOBPCG and ARPACK
|
||||
// eigenvalue solvers together with the BoomerAMG preconditioner
|
||||
// in HYPRE, as well as optionally the SuperLU parallel direct
|
||||
// solver. Reusing a single GLVis visualization window for
|
||||
// multiple eigenfunctions is also illustrated.
|
||||
//
|
||||
// We recommend viewing Example 1 before viewing this example.
|
||||
|
||||
#include "mfem.hpp"
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
|
||||
using namespace std;
|
||||
using namespace mfem;
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
// 1. Initialize MPI.
|
||||
int num_procs, myid;
|
||||
MPI_Init(&argc, &argv);
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
|
||||
|
||||
// 2. Parse command-line options.
|
||||
const char *mesh_file = "../data/star.mesh";
|
||||
int ser_ref_levels = 2;
|
||||
int par_ref_levels = 1;
|
||||
int order = 1;
|
||||
int nev = 5;
|
||||
bool slu_solver = false;
|
||||
bool use_arpack = false;
|
||||
bool visualization = 1;
|
||||
|
||||
OptionsParser args(argc, argv);
|
||||
args.AddOption(&mesh_file, "-m", "--mesh",
|
||||
"Mesh file to use.");
|
||||
args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
|
||||
"Number of times to refine the mesh uniformly in serial.");
|
||||
args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
|
||||
"Number of times to refine the mesh uniformly in parallel.");
|
||||
args.AddOption(&order, "-o", "--order",
|
||||
"Finite element order (polynomial degree) or -1 for"
|
||||
" isoparametric space.");
|
||||
args.AddOption(&nev, "-n", "--num-eigs",
|
||||
"Number of desired eigenmodes.");
|
||||
#ifdef MFEM_USE_SUPERLU
|
||||
args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
|
||||
"--no-superlu", "Use the SuperLU Solver.");
|
||||
#endif
|
||||
#ifdef MFEM_USE_ARPACK
|
||||
args.AddOption(&use_arpack, "-arpack", "--use-arpack", "-no-arpack",
|
||||
"--no-arpack",
|
||||
"Enable or disable the use of ARPACK.");
|
||||
#endif
|
||||
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
|
||||
"--no-visualization",
|
||||
"Enable or disable GLVis visualization.");
|
||||
args.Parse();
|
||||
if (!args.Good())
|
||||
{
|
||||
if (myid == 0)
|
||||
{
|
||||
args.PrintUsage(cout);
|
||||
}
|
||||
MPI_Finalize();
|
||||
return 1;
|
||||
}
|
||||
if (myid == 0)
|
||||
{
|
||||
args.PrintOptions(cout);
|
||||
}
|
||||
|
||||
// 3. Read the (serial) mesh from the given mesh file on all processors. We
|
||||
// can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
|
||||
// and volume meshes with the same code.
|
||||
Mesh *mesh;
|
||||
ifstream imesh(mesh_file);
|
||||
if (!imesh)
|
||||
{
|
||||
if (myid == 0)
|
||||
{
|
||||
cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
|
||||
}
|
||||
MPI_Finalize();
|
||||
return 2;
|
||||
}
|
||||
mesh = new Mesh(imesh, 1, 1);
|
||||
imesh.close();
|
||||
int dim = mesh->Dimension();
|
||||
|
||||
// 4. Refine the serial mesh on all processors to increase the resolution. In
|
||||
// this example we do 'ref_levels' of uniform refinement (2 by default, or
|
||||
// specified on the command line with -rs).
|
||||
for (int lev = 0; lev < ser_ref_levels; lev++)
|
||||
{
|
||||
mesh->UniformRefinement();
|
||||
}
|
||||
|
||||
// 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
|
||||
// this mesh further in parallel to increase the resolution (1 time by
|
||||
// default, or specified on the command line with -rp). Once the parallel
|
||||
// mesh is defined, the serial mesh can be deleted.
|
||||
ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
|
||||
delete mesh;
|
||||
for (int lev = 0; lev < par_ref_levels; lev++)
|
||||
{
|
||||
pmesh->UniformRefinement();
|
||||
}
|
||||
|
||||
// 6. Define a parallel finite element space on the parallel mesh. Here we
|
||||
// use continuous Lagrange finite elements of the specified order. If
|
||||
// order < 1, we instead use an isoparametric/isogeometric space.
|
||||
FiniteElementCollection *fec;
|
||||
if (order > 0)
|
||||
{
|
||||
fec = new H1_FECollection(order, dim);
|
||||
}
|
||||
else if (pmesh->GetNodes())
|
||||
{
|
||||
fec = pmesh->GetNodes()->OwnFEC();
|
||||
}
|
||||
else
|
||||
{
|
||||
fec = new H1_FECollection(order = 1, dim);
|
||||
}
|
||||
ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
|
||||
HYPRE_Int size = fespace->GlobalTrueVSize();
|
||||
if (myid == 0)
|
||||
{
|
||||
cout << "Number of unknowns: " << size << endl;
|
||||
}
|
||||
|
||||
// 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
|
||||
// element space. The first corresponds to the Laplacian operator -Delta,
|
||||
// while the second is a simple mass matrix needed on the right hand side
|
||||
// of the generalized eigenvalue problem below. The boundary conditions
|
||||
// are implemented by elimination with special values on the diagonal to
|
||||
// shift the Dirichlet eigenvalues out of the computational range. After
|
||||
// serial and parallel assembly we extract the corresponding parallel
|
||||
// matrices A and M.
|
||||
ConstantCoefficient one(1.0);
|
||||
Array<int> ess_bdr;
|
||||
if (pmesh->bdr_attributes.Size())
|
||||
{
|
||||
ess_bdr.SetSize(pmesh->bdr_attributes.Max());
|
||||
ess_bdr = 1;
|
||||
}
|
||||
|
||||
ParBilinearForm *a = new ParBilinearForm(fespace);
|
||||
a->AddDomainIntegrator(new DiffusionIntegrator(one));
|
||||
if (pmesh->bdr_attributes.Size() == 0)
|
||||
{
|
||||
// Add a mass term if the mesh has no boundary, e.g. periodic mesh or
|
||||
// closed surface.
|
||||
a->AddDomainIntegrator(new MassIntegrator(one));
|
||||
}
|
||||
a->Assemble();
|
||||
a->EliminateEssentialBCDiag(ess_bdr, 1.0);
|
||||
a->Finalize();
|
||||
|
||||
ParBilinearForm *m = new ParBilinearForm(fespace);
|
||||
m->AddDomainIntegrator(new MassIntegrator(one));
|
||||
m->Assemble();
|
||||
// shift the eigenvalue corresponding to eliminated dofs to a large value
|
||||
m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
|
||||
m->Finalize();
|
||||
|
||||
HypreParMatrix *A = a->ParallelAssemble();
|
||||
HypreParMatrix *M = m->ParallelAssemble();
|
||||
|
||||
#ifdef MFEM_USE_SUPERLU
|
||||
Operator * Arow = NULL;
|
||||
if (slu_solver)
|
||||
{
|
||||
Arow = new SuperLURowLocMatrix(*A);
|
||||
}
|
||||
#endif
|
||||
|
||||
delete a;
|
||||
delete m;
|
||||
|
||||
// 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
|
||||
// preconditioner for A to be used within the solver. Set the matrices
|
||||
// which define the generalized eigenproblem A x = lambda M x.
|
||||
Eigensolver * esolver = NULL;
|
||||
Solver * solver = NULL;
|
||||
Solver * precond = NULL;
|
||||
|
||||
if (!slu_solver)
|
||||
{
|
||||
HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
|
||||
amg->SetPrintLevel(0);
|
||||
precond = amg;
|
||||
|
||||
#ifdef MFEM_USE_ARPACK
|
||||
if ( use_arpack )
|
||||
{
|
||||
HyprePCG * pcg = new HyprePCG(*A);
|
||||
pcg->SetTol(1e-12);
|
||||
pcg->SetMaxIter(200);
|
||||
pcg->SetPreconditioner(*amg);
|
||||
pcg->SetPrintLevel(0);
|
||||
|
||||
solver = pcg;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#ifdef MFEM_USE_SUPERLU
|
||||
else
|
||||
{
|
||||
SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
|
||||
superlu->SetPrintStatistics(false);
|
||||
superlu->SetSymmetricPattern(true);
|
||||
superlu->SetColumnPermutation(superlu::PARMETIS);
|
||||
superlu->SetOperator(*Arow);
|
||||
|
||||
solver = use_arpack?superlu:NULL;
|
||||
precond = use_arpack?NULL:superlu;
|
||||
}
|
||||
#endif
|
||||
|
||||
if ( use_arpack )
|
||||
{
|
||||
ParArPackSym * arpack = new ParArPackSym(MPI_COMM_WORLD);
|
||||
arpack->SetMode(3);
|
||||
arpack->SetPrintLevel(2);
|
||||
arpack->SetSolver(*solver);
|
||||
|
||||
esolver = arpack;
|
||||
}
|
||||
else
|
||||
{
|
||||
HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
|
||||
lobpcg->SetPreconditioner(*precond);
|
||||
lobpcg->SetPrecondUsageMode(1);
|
||||
lobpcg->SetPrintLevel(1);
|
||||
|
||||
esolver = lobpcg;
|
||||
}
|
||||
|
||||
esolver->SetNumModes(nev);
|
||||
esolver->SetMaxIter(100);
|
||||
esolver->SetTol(1e-8);
|
||||
|
||||
esolver->SetMassMatrix(*M);
|
||||
esolver->SetOperator(*A);
|
||||
|
||||
// 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
|
||||
// parallel grid function to represent each of the eigenmodes returned by
|
||||
// the solver.
|
||||
Array<double> eigenvalues;
|
||||
esolver->Solve();
|
||||
esolver->GetEigenvalues(eigenvalues);
|
||||
|
||||
if ( myid == 0 && use_arpack )
|
||||
{
|
||||
cout << endl;
|
||||
for (int i=0; i<eigenvalues.Size(); i++)
|
||||
{
|
||||
cout << "Eigenvalue lambda " << eigenvalues[i] << endl;
|
||||
}
|
||||
cout << endl;
|
||||
}
|
||||
|
||||
ParGridFunction x(fespace);
|
||||
|
||||
// 10. Save the refined mesh and the modes in parallel. This output can be
|
||||
// viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
|
||||
{
|
||||
ostringstream mesh_name, mode_name;
|
||||
mesh_name << "mesh." << setfill('0') << setw(6) << myid;
|
||||
|
||||
ofstream mesh_ofs(mesh_name.str().c_str());
|
||||
mesh_ofs.precision(8);
|
||||
pmesh->Print(mesh_ofs);
|
||||
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x.Distribute(esolver->GetEigenvector(i));
|
||||
|
||||
mode_name << "mode_" << setfill('0') << setw(2) << i << "."
|
||||
<< setfill('0') << setw(6) << myid;
|
||||
|
||||
ofstream mode_ofs(mode_name.str().c_str());
|
||||
mode_ofs.precision(8);
|
||||
x.Save(mode_ofs);
|
||||
mode_name.str("");
|
||||
}
|
||||
}
|
||||
|
||||
// 11. Send the solution by socket to a GLVis server.
|
||||
if (visualization)
|
||||
{
|
||||
char vishost[] = "localhost";
|
||||
int visport = 19916;
|
||||
socketstream mode_sock(vishost, visport);
|
||||
mode_sock.precision(8);
|
||||
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
if ( myid == 0 )
|
||||
{
|
||||
cout << "Eigenmode " << i+1 << '/' << nev
|
||||
<< ", Lambda = " << eigenvalues[i] << endl;
|
||||
}
|
||||
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x.Distribute(esolver->GetEigenvector(i));
|
||||
|
||||
mode_sock << "parallel " << num_procs << " " << myid << "\n"
|
||||
<< "solution\n" << *pmesh << x << flush
|
||||
<< "window_title 'Eigenmode " << i+1 << '/' << nev
|
||||
<< ", Lambda = " << eigenvalues[i] << "'" << endl;
|
||||
|
||||
char c;
|
||||
if (myid == 0)
|
||||
{
|
||||
cout << "press (q)uit or (c)ontinue --> " << flush;
|
||||
cin >> c;
|
||||
}
|
||||
MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
|
||||
|
||||
if (c != 'c')
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
mode_sock.close();
|
||||
}
|
||||
|
||||
// 12. Free the used memory.
|
||||
delete esolver;
|
||||
delete solver;
|
||||
delete precond;
|
||||
delete M;
|
||||
delete A;
|
||||
|
||||
delete fespace;
|
||||
if (order > 0)
|
||||
{
|
||||
delete fec;
|
||||
}
|
||||
delete pmesh;
|
||||
|
||||
MPI_Finalize();
|
||||
|
||||
return 0;
|
||||
}
|
||||
+5
-5
@@ -276,8 +276,8 @@ int main(int argc, char *argv[])
|
||||
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = lobpcg->GetEigenvector(i);
|
||||
// convert eigenvector from Vector to ParGridFunction
|
||||
x.Distribute(lobpcg->GetEigenvector(i));
|
||||
|
||||
mode_name << "mode_" << setfill('0') << setw(2) << i << "."
|
||||
<< setfill('0') << setw(6) << myid;
|
||||
@@ -303,7 +303,7 @@ int main(int argc, char *argv[])
|
||||
pmesh->Print(adios2output);
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
x = lobpcg->GetEigenvector(i);
|
||||
x.Distribute(lobpcg->GetEigenvector(i));
|
||||
// x is a temporary that must be saved immediately
|
||||
x.Save(adios2output, "mode_" + std::to_string(i));
|
||||
}
|
||||
@@ -326,8 +326,8 @@ int main(int argc, char *argv[])
|
||||
<< ", Lambda = " << eigenvalues[i] << endl;
|
||||
}
|
||||
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = lobpcg->GetEigenvector(i);
|
||||
// convert eigenvector from Vector to ParGridFunction
|
||||
x.Distribute(lobpcg->GetEigenvector(i));
|
||||
|
||||
mode_sock << "parallel " << num_procs << " " << myid << "\n"
|
||||
<< "solution\n" << *pmesh << x << flush
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
// MFEM Example 13
|
||||
//
|
||||
// Compile with: make ex3p
|
||||
//
|
||||
// Sample runs: ex13 -m ../data/star.mesh -s 5
|
||||
// ex13 -m ../data/square-disc.mesh -o 2 -n 4 // minres fails to conv.
|
||||
// ex13 -m ../data/beam-hex.mesh
|
||||
// ex13 -m ../data/square-disc.mesh -rs 1 -s 26
|
||||
// ex13 -m ../data/square-disc-nurbs.mesh -rs 3 -s 26
|
||||
// ex13 -m ../data/amr-quad.mesh -o 2 // minres fails to conv.
|
||||
// ex13 -m ../data/mobius-strip.mesh -n 8
|
||||
//
|
||||
// Description: This example code solves a simple 3D electromagnetic
|
||||
// eigenmode problem corresponding to the second order
|
||||
// Maxwell equation curl curl E = lambda E with boundary
|
||||
// condition E x n = 0. We discretize with Nedelec finite
|
||||
// elements.
|
||||
//
|
||||
// The example demonstrates the use of H(curl) finite element
|
||||
// spaces with the curl-curl and the (vector finite element) mass
|
||||
// bilinear form, as well as the use of the ARPACK eigenmode
|
||||
// solver for symmetric matrices using the shift-invert mode.
|
||||
//
|
||||
|
||||
#include "mfem.hpp"
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
|
||||
using namespace std;
|
||||
using namespace mfem;
|
||||
|
||||
#ifdef MFEM_USE_ARPACK
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
// 1. Parse command-line options.
|
||||
const char *mesh_file = "../data/beam-tet.mesh";
|
||||
int order = 1;
|
||||
int nev = 5;
|
||||
int sr = 2;
|
||||
double sigma = 11.0;
|
||||
bool visualization = 1;
|
||||
bool arp_solver = true;
|
||||
|
||||
OptionsParser args(argc, argv);
|
||||
args.AddOption(&mesh_file, "-m", "--mesh",
|
||||
"Mesh file to use.");
|
||||
args.AddOption(&order, "-o", "--order",
|
||||
"Finite element order (polynomial degree).");
|
||||
args.AddOption(&nev, "-n", "--num-eigs",
|
||||
"Number of desired eigenmodes.");
|
||||
args.AddOption(&sr, "-rs", "--refine-serial",
|
||||
"Number of times to refine the mesh uniformly in serial.");
|
||||
args.AddOption(&sigma, "-s", "--shift",
|
||||
"Average of the desired eigenvalue range.");
|
||||
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
|
||||
"--no-visualization",
|
||||
"Enable or disable GLVis visualization.");
|
||||
args.Parse();
|
||||
if (!args.Good())
|
||||
{
|
||||
args.PrintUsage(cout);
|
||||
return 1;
|
||||
}
|
||||
args.PrintOptions(cout);
|
||||
|
||||
// 2. Read the mesh from the given mesh file. We can handle triangular,
|
||||
// quadrilateral, tetrahedral, hexahedral, surface and volume meshes
|
||||
// with the same code.
|
||||
Mesh *mesh;
|
||||
ifstream imesh(mesh_file);
|
||||
if (!imesh)
|
||||
{
|
||||
cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
|
||||
return 2;
|
||||
}
|
||||
mesh = new Mesh(imesh, 1, 1);
|
||||
imesh.close();
|
||||
int dim = mesh->Dimension();
|
||||
|
||||
// 3. Refine the mesh to increase the resolution. In this example we do
|
||||
// 'ref_levels' of uniform refinement.
|
||||
{
|
||||
int ref_levels = sr;
|
||||
for (int l = 0; l < ref_levels; l++)
|
||||
{
|
||||
mesh->UniformRefinement();
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Define a finite element space on the mesh. Here we use the lowest
|
||||
// order Nedelec finite elements, but we can easily switch
|
||||
// to higher-order spaces by changing the value of p.
|
||||
FiniteElementCollection *fec = new ND_FECollection(order, dim);
|
||||
FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
|
||||
int size = fespace->GetVSize();
|
||||
|
||||
cout << "Number of unknowns: " << size << endl;
|
||||
cout << "Number of boundary attributes: " << mesh->bdr_attributes.Max()
|
||||
<< endl;
|
||||
|
||||
// 5. Set up the parallel bilinear form corresponding to the EM diffusion
|
||||
// operator curl muinv curl - sigma I, by adding the curl-curl and the
|
||||
// mass domain integrators and finally imposing homogeneous Dirichlet
|
||||
// boundary conditions. The boundary conditions are implemented by
|
||||
// marking all the boundary attributes from the mesh as essential
|
||||
// (Dirichlet). After serial and parallel assembly we extract the
|
||||
// parallel matrices A and M.
|
||||
Coefficient *muinv = new ConstantCoefficient(1.0);
|
||||
Coefficient *negSigma = new ConstantCoefficient(-sigma);
|
||||
|
||||
BilinearForm *a = new BilinearForm(fespace);
|
||||
a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
|
||||
a->AddDomainIntegrator(new VectorFEMassIntegrator(*negSigma));
|
||||
a->Assemble();
|
||||
Array<int> ess_bdr(mesh->bdr_attributes.Max());
|
||||
ess_bdr = 1;
|
||||
a->EliminateEssentialBC(ess_bdr);
|
||||
a->Finalize();
|
||||
|
||||
BilinearForm *m = new BilinearForm(fespace);
|
||||
m->AddDomainIntegrator(new VectorFEMassIntegrator());
|
||||
m->Assemble();
|
||||
m->EliminateEssentialBCDiag(ess_bdr, sqrt(numeric_limits<double>::min()));
|
||||
m->Finalize();
|
||||
|
||||
// 6. Define a parallel grid function to approximate each of the
|
||||
// eigenmodes returned by the solver. Use this as a template to
|
||||
// create a special multi-vector object needed by the eigensolver
|
||||
// which is then initialized with random values.
|
||||
GridFunction x(fespace);
|
||||
x = 0.0;
|
||||
|
||||
// 7. Define and configure the GMRES
|
||||
// solver to be used within the eigensolver.
|
||||
Solver * solver = NULL;
|
||||
if ( false )
|
||||
{
|
||||
GMRESSolver * gmres = new GMRESSolver();
|
||||
|
||||
gmres->SetOperator(*a);
|
||||
gmres->SetRelTol(1e-8);
|
||||
gmres->SetMaxIter(1000);
|
||||
gmres->SetPrintLevel(0);
|
||||
solver = gmres;
|
||||
}
|
||||
else
|
||||
{
|
||||
#ifndef MFEM_USE_SUITESPARSE
|
||||
cout << "Building MINRESSolver" << endl;
|
||||
MINRESSolver * minres = new MINRESSolver();
|
||||
|
||||
minres->SetRelTol(1e-12);
|
||||
minres->SetMaxIter(1000);
|
||||
minres->SetPrintLevel(0);
|
||||
solver = minres;
|
||||
#else
|
||||
// 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
|
||||
cout << "Building UMFPackSolver" << endl;
|
||||
UMFPackSolver * umf_solver = new UMFPackSolver;
|
||||
umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
|
||||
solver = umf_solver;
|
||||
#endif
|
||||
}
|
||||
solver->SetOperator(a->SpMat());
|
||||
|
||||
// 7. Define and configure the ARPACK eigensolver
|
||||
SymGenEigensolver * eig_solver = NULL;
|
||||
if (arp_solver)
|
||||
{
|
||||
ArPackSAUPD * arpack = new ArPackSAUPD();
|
||||
|
||||
arpack->SetNumModes(nev);
|
||||
arpack->SetMaxIter(400);
|
||||
arpack->SetTol(1e-8);
|
||||
arpack->SetShift(sigma);
|
||||
arpack->SetMode(3);
|
||||
arpack->SetPrintLevel(2);
|
||||
arpack->SetSolver(*solver);
|
||||
|
||||
eig_solver = arpack;
|
||||
}
|
||||
|
||||
eig_solver->SetOperators(*a, *m);
|
||||
|
||||
// Obtain the eigenvalues and eigenvectors
|
||||
Array<double> eigenvalues(nev);
|
||||
eigenvalues = -1.0;
|
||||
|
||||
// arpack->Solve(eigenvalues, *eigenvectors);
|
||||
eig_solver->Solve();
|
||||
|
||||
eig_solver->GetEigenvalues(eigenvalues);
|
||||
|
||||
cout << endl;
|
||||
std::ios::fmtflags old_fmt = cout.flags();
|
||||
cout.setf(std::ios::scientific);
|
||||
std::streamsize old_prec = cout.precision(14);
|
||||
for (int i=0; i<min(nev,eigenvalues.Size()); i++)
|
||||
{
|
||||
cout << "Eigenvalue lambda " << eigenvalues[i] << endl;
|
||||
}
|
||||
cout.precision(old_prec);
|
||||
cout.flags(old_fmt);
|
||||
cout << endl;
|
||||
|
||||
VisItDataCollection visit_dc("Example13", mesh);
|
||||
GridFunction ** mode = new GridFunction*[min(nev,eigenvalues.Size())];
|
||||
for (int i=0; i<min(nev,eigenvalues.Size()); i++)
|
||||
{
|
||||
mode[i] = new GridFunction(fespace);
|
||||
*mode[i] = eig_solver->GetEigenvector(i);
|
||||
|
||||
ostringstream modeName;
|
||||
modeName << "mode_" << setfill('0') << setw(2) << i;
|
||||
visit_dc.RegisterField(modeName.str().c_str(),mode[i]);
|
||||
}
|
||||
visit_dc.Save();
|
||||
|
||||
// 8. Save the refined mesh and the modes. This output can
|
||||
// be viewed later using GLVis: "glvis -m mesh -g mode".
|
||||
{
|
||||
ofstream mesh_ofs("refined.mesh");
|
||||
mesh_ofs.precision(8);
|
||||
mesh->Print(mesh_ofs);
|
||||
|
||||
for (int i=0; i<min(nev,eigenvalues.Size()); i++)
|
||||
{
|
||||
x = eig_solver->GetEigenvector(i);
|
||||
|
||||
ostringstream modeName;
|
||||
modeName << "mode_" << setfill('0') << setw(2) << i;
|
||||
|
||||
ofstream mode_ofs(modeName.str().c_str());
|
||||
mode_ofs.precision(8);
|
||||
x.Save(mode_ofs);
|
||||
modeName.str("");
|
||||
}
|
||||
}
|
||||
|
||||
// 9. Send the solution by socket to a GLVis server.
|
||||
if (visualization)
|
||||
{
|
||||
char vishost[] = "localhost";
|
||||
int visport = 19916;
|
||||
socketstream mode_sock(vishost, visport);
|
||||
mode_sock.precision(8);
|
||||
|
||||
for (int i=0; i<min(nev,eigenvalues.Size()); i++)
|
||||
{
|
||||
x = eig_solver->GetEigenvector(i);
|
||||
|
||||
mode_sock << "solution\n" << *mesh << x << flush;
|
||||
|
||||
char c;
|
||||
cout << "press (q)uit or (c)ontinue --> " << flush;
|
||||
cin >> c;
|
||||
|
||||
if (c != 'c')
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
mode_sock.close();
|
||||
}
|
||||
|
||||
// 10. Free the used memory.
|
||||
delete a;
|
||||
delete m;
|
||||
delete negSigma;
|
||||
delete muinv;
|
||||
delete eig_solver;
|
||||
delete solver;
|
||||
// delete X;
|
||||
delete fespace;
|
||||
delete fec;
|
||||
delete mesh;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // MFEM_USE_ARPACK
|
||||
+4
-4
@@ -215,8 +215,8 @@ int main(int argc, char *argv[])
|
||||
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = ame->GetEigenvector(i);
|
||||
// convert eigenvector from Vector to ParGridFunction
|
||||
x.Distribute(ame->GetEigenvector(i));
|
||||
|
||||
mode_name << "mode_" << setfill('0') << setw(2) << i << "."
|
||||
<< setfill('0') << setw(6) << myid;
|
||||
@@ -244,8 +244,8 @@ int main(int argc, char *argv[])
|
||||
<< ", Lambda = " << eigenvalues[i] << endl;
|
||||
}
|
||||
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = ame->GetEigenvector(i);
|
||||
// convert eigenvector from Vector to ParGridFunction
|
||||
x.Distribute(ame->GetEigenvector(i));
|
||||
|
||||
mode_sock << "parallel " << num_procs << " " << myid << "\n"
|
||||
<< "solution\n" << *pmesh << x << flush
|
||||
|
||||
+34
-60
@@ -42,11 +42,7 @@
|
||||
// mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
|
||||
// mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
|
||||
// mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
|
||||
// mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/beam-tet.mesh
|
||||
//
|
||||
// Device simplices sample runs:
|
||||
// mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tet.mesh
|
||||
// mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tri.mesh
|
||||
// mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
|
||||
//
|
||||
// Description: This example code demonstrates the use of MFEM to define a
|
||||
// simple finite element discretization of the Poisson problem
|
||||
@@ -87,9 +83,6 @@ int main(int argc, char *argv[])
|
||||
const char *device_config = "cpu";
|
||||
bool visualization = true;
|
||||
bool algebraic_ceed = false;
|
||||
#ifdef MFEM_USE_CUDSS
|
||||
bool cudss_solver = false;
|
||||
#endif
|
||||
|
||||
OptionsParser args(argc, argv);
|
||||
args.AddOption(&mesh_file, "-m", "--mesh",
|
||||
@@ -109,10 +102,6 @@ int main(int argc, char *argv[])
|
||||
args.AddOption(&algebraic_ceed, "-a", "--algebraic",
|
||||
"-no-a", "--no-algebraic",
|
||||
"Use algebraic Ceed solver");
|
||||
#endif
|
||||
#ifdef MFEM_USE_CUDSS
|
||||
args.AddOption(&cudss_solver, "-cudss", "--cudss-solver", "-no-cudss",
|
||||
"--no-cudss-solver", "Use the cuDSS Solver.");
|
||||
#endif
|
||||
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
|
||||
"--no-visualization",
|
||||
@@ -169,20 +158,19 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
// 7. Define a parallel finite element space on the parallel mesh. Here we
|
||||
// use continuous Lagrange finite elements of the specified order.
|
||||
// - If order < 1, we instead use an isoparametric/isogeometric space.
|
||||
// - If the mesh is simplicial and partial assembly is requested,
|
||||
// we use the positive basis, which supports device execution.
|
||||
// use continuous Lagrange finite elements of the specified order. If
|
||||
// order < 1, we instead use an isoparametric/isogeometric space.
|
||||
FiniteElementCollection *fec;
|
||||
auto basis_type = (pa && pmesh.IsSimplexMesh()) ?
|
||||
BasisType::Positive : BasisType::GaussLobatto;
|
||||
bool delete_fec;
|
||||
if (order > 0)
|
||||
{
|
||||
fec = new H1_FECollection(order, dim, basis_type);
|
||||
fec = new H1_FECollection(order, dim);
|
||||
delete_fec = true;
|
||||
}
|
||||
else if (pmesh.GetNodes())
|
||||
{
|
||||
fec = pmesh.GetNodes()->OwnFEC();
|
||||
delete_fec = false;
|
||||
if (myid == 0)
|
||||
{
|
||||
cout << "Using isoparametric FEs: " << fec->Name() << endl;
|
||||
@@ -190,7 +178,8 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
else
|
||||
{
|
||||
fec = new H1_FECollection(order = 1, dim, basis_type);
|
||||
fec = new H1_FECollection(order = 1, dim);
|
||||
delete_fec = true;
|
||||
}
|
||||
ParFiniteElementSpace fespace(&pmesh, fec);
|
||||
HYPRE_BigInt size = fespace.GlobalTrueVSize();
|
||||
@@ -259,51 +248,33 @@ int main(int argc, char *argv[])
|
||||
// 13. Solve the linear system A X = B.
|
||||
// * With full assembly, use the BoomerAMG preconditioner from hypre.
|
||||
// * With partial assembly, use Jacobi smoothing, for now.
|
||||
#ifdef MFEM_USE_CUDSS
|
||||
if (!pa && (Device::Allows(Backend::CUDA_MASK) && cudss_solver))
|
||||
Solver *prec = NULL;
|
||||
if (pa)
|
||||
{
|
||||
// Solve using a direct solver with cuDSS
|
||||
CuDSSSolver cudss_solver(MPI_COMM_WORLD);
|
||||
cudss_solver.SetMatrixSymType(
|
||||
CuDSSSolver::SYMMETRIC_POSITIVE_DEFINITE);
|
||||
cudss_solver.SetMatrixViewType(CuDSSSolver::UPPER);
|
||||
cudss_solver.SetOperator(*A);
|
||||
cudss_solver.Mult(B, X);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
Solver *prec = NULL;
|
||||
if (pa)
|
||||
if (UsesTensorBasis(fespace))
|
||||
{
|
||||
if (UsesTensorBasis(fespace))
|
||||
if (algebraic_ceed)
|
||||
{
|
||||
if (algebraic_ceed)
|
||||
{
|
||||
prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
|
||||
}
|
||||
else
|
||||
{
|
||||
prec = new OperatorJacobiSmoother(a, ess_tdof_list);
|
||||
}
|
||||
prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
|
||||
}
|
||||
else
|
||||
{
|
||||
prec = new OperatorJacobiSmoother(a, ess_tdof_list);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
prec = new HypreBoomerAMG;
|
||||
}
|
||||
CGSolver cg(MPI_COMM_WORLD);
|
||||
cg.SetRelTol(1e-12);
|
||||
cg.SetMaxIter(2000);
|
||||
cg.SetPrintLevel(1);
|
||||
if (prec)
|
||||
{
|
||||
cg.SetPreconditioner(*prec);
|
||||
}
|
||||
cg.SetOperator(*A);
|
||||
cg.Mult(B, X);
|
||||
delete prec;
|
||||
}
|
||||
else
|
||||
{
|
||||
prec = new HypreBoomerAMG;
|
||||
}
|
||||
CGSolver cg(MPI_COMM_WORLD);
|
||||
cg.SetRelTol(1e-12);
|
||||
cg.SetMaxIter(2000);
|
||||
cg.SetPrintLevel(1);
|
||||
if (prec) { cg.SetPreconditioner(*prec); }
|
||||
cg.SetOperator(*A);
|
||||
cg.Mult(B, X);
|
||||
delete prec;
|
||||
|
||||
// 14. Recover the parallel grid function corresponding to X. This is the
|
||||
// local finite element solution on each processor.
|
||||
@@ -337,7 +308,10 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
// 17. Free the used memory.
|
||||
if (order > 0) { delete fec; }
|
||||
if (delete_fec)
|
||||
{
|
||||
delete fec;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -95,15 +95,6 @@ int main(int argc, char *argv[])
|
||||
args.PrintOptions(cout);
|
||||
}
|
||||
|
||||
if (amg_elast && !static_cond && reorder_space)
|
||||
{
|
||||
if (myid == 0)
|
||||
cerr << "\nThe AMG elasticity solver requires ordering byVDIM! "
|
||||
<< "Ignoring the specified option -nodes/--by-nodes.\n"
|
||||
<< endl;
|
||||
reorder_space = false;
|
||||
}
|
||||
|
||||
// 3. Enable hardware devices such as GPUs, and programming models such as
|
||||
// CUDA, OCCA, RAJA and OpenMP based on command line options.
|
||||
Device device(device_config);
|
||||
|
||||
+4
-4
@@ -228,7 +228,7 @@ int main(int argc, char *argv[])
|
||||
for (int i=0; i<nev; i++)
|
||||
{
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = ame->GetEigenvector(i);
|
||||
x.Distribute(ame->GetEigenvector(i));
|
||||
curl.Mult(x, dx);
|
||||
|
||||
mode_name << "mode_" << setfill('0') << setw(2) << i << "."
|
||||
@@ -295,7 +295,7 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = ame->GetEigenvector(i);
|
||||
x.Distribute(ame->GetEigenvector(i));
|
||||
curl.Mult(x, dx);
|
||||
|
||||
{
|
||||
@@ -469,7 +469,7 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = ame->GetEigenvector(i);
|
||||
x.Distribute(ame->GetEigenvector(i));
|
||||
curl.Mult(x, dx);
|
||||
|
||||
{
|
||||
@@ -599,7 +599,7 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
// convert eigenvector from HypreParVector to ParGridFunction
|
||||
x = ame->GetEigenvector(i);
|
||||
x.Distribute(ame->GetEigenvector(i));
|
||||
curl.Mult(x, dx);
|
||||
|
||||
mode_sock << "parallel " << num_procs << " " << myid << "\n"
|
||||
|
||||
+3
-3
@@ -658,7 +658,7 @@ void ScalarWaveGuide(int mode, ParGridFunction &x)
|
||||
lobpcg.SetOperator(*A);
|
||||
lobpcg.Solve();
|
||||
|
||||
x = lobpcg.GetEigenvector(mode);
|
||||
x.Distribute(lobpcg.GetEigenvector(mode));
|
||||
|
||||
delete A;
|
||||
delete M;
|
||||
@@ -714,7 +714,7 @@ void VectorWaveGuide(int mode, ParGridFunction &x)
|
||||
ame.SetOperator(*A);
|
||||
ame.Solve();
|
||||
|
||||
x = ame.GetEigenvector(mode);
|
||||
x.Distribute(ame.GetEigenvector(mode));
|
||||
|
||||
delete A;
|
||||
delete M;
|
||||
@@ -780,7 +780,7 @@ void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
|
||||
lobpcg.SetOperator(*A);
|
||||
lobpcg.Solve();
|
||||
|
||||
x = lobpcg.GetEigenvector(mode);
|
||||
x.Distribute(lobpcg.GetEigenvector(mode));
|
||||
|
||||
x_l2.ProjectCoefficient(xCoef);
|
||||
|
||||
|
||||
@@ -76,4 +76,4 @@ clean-build:
|
||||
rm -rf *.dSYM *.TVD.*breakpoints
|
||||
|
||||
clean-exec:
|
||||
@rm -f refined.mesh sol.gf mesh.* sol.*
|
||||
@rm -f refined.mesh sol.gf
|
||||
|
||||
@@ -1,548 +0,0 @@
|
||||
#include <mfem.hpp>
|
||||
|
||||
#ifdef MFEM_USE_PROTEUS
|
||||
|
||||
#include "../fem/dfem/util.hpp"
|
||||
|
||||
#include <proteus/CppJitModule.h>
|
||||
|
||||
#include "jitplayground.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <fstream>
|
||||
#include <initializer_list>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace util
|
||||
{
|
||||
constexpr std::string_view Dirname(std::string_view path)
|
||||
{
|
||||
const size_t last_sep = path.find_last_of("/\\");
|
||||
if (last_sep == std::string_view::npos) { return {}; }
|
||||
return path.substr(0, last_sep);
|
||||
}
|
||||
|
||||
constexpr std::string_view thisFileDir = Dirname(__FILE__);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static std::string TypeNameString()
|
||||
{
|
||||
return std::string(mfem::future::get_type_name<T>());
|
||||
}
|
||||
|
||||
template <typename Tuple, size_t... Is>
|
||||
static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
|
||||
{
|
||||
return std::array<std::string, sizeof...(Is)>
|
||||
{
|
||||
TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
|
||||
};
|
||||
}
|
||||
|
||||
template <typename Tuple>
|
||||
static auto ParamTypeStrings()
|
||||
{
|
||||
return ParamTypeStringsImpl<Tuple>(
|
||||
std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
|
||||
}
|
||||
|
||||
static std::string_view Trim(std::string_view s)
|
||||
{
|
||||
size_t begin = 0;
|
||||
while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
|
||||
{
|
||||
++begin;
|
||||
}
|
||||
size_t end = s.size();
|
||||
while (end > begin &&
|
||||
std::isspace(static_cast<unsigned char>(s[end - 1])))
|
||||
{
|
||||
--end;
|
||||
}
|
||||
return s.substr(begin, end - begin);
|
||||
}
|
||||
|
||||
static bool IsValidIdentifier(std::string_view s)
|
||||
{
|
||||
if (s.empty()) { return false; }
|
||||
const unsigned char c0 = static_cast<unsigned char>(s[0]);
|
||||
if (!(std::isalpha(c0) || c0 == '_')) { return false; }
|
||||
for (size_t i = 1; i < s.size(); ++i)
|
||||
{
|
||||
const unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
if (!(std::isalnum(c) || c == '_')) { return false; }
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ParseJitDirective(std::string_view line,
|
||||
std::string &type,
|
||||
std::string &var,
|
||||
std::string &kind)
|
||||
{
|
||||
const size_t jit_pos = line.find("$JIT");
|
||||
if (jit_pos == std::string_view::npos) { return false; }
|
||||
|
||||
const size_t open = line.find('[', jit_pos);
|
||||
const size_t close = line.find(']', jit_pos);
|
||||
MFEM_VERIFY(open != std::string_view::npos &&
|
||||
close != std::string_view::npos &&
|
||||
close > open,
|
||||
"malformed $JIT directive (expected brackets): " << line);
|
||||
|
||||
const std::string_view payload = line.substr(open + 1, close - open - 1);
|
||||
const size_t comma1 = payload.find(',');
|
||||
const size_t comma2 = (comma1 == std::string_view::npos)
|
||||
? std::string_view::npos
|
||||
: payload.find(',', comma1 + 1);
|
||||
MFEM_VERIFY(comma1 != std::string_view::npos &&
|
||||
comma2 != std::string_view::npos,
|
||||
"malformed $JIT directive (expected 3 comma-separated fields): "
|
||||
<< line);
|
||||
|
||||
const std::string_view f0 = Trim(payload.substr(0, comma1));
|
||||
const std::string_view f1 = Trim(payload.substr(comma1 + 1,
|
||||
comma2 - comma1 - 1));
|
||||
const std::string_view f2 = Trim(payload.substr(comma2 + 1));
|
||||
MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
|
||||
"malformed $JIT directive (empty field): " << line);
|
||||
|
||||
type.assign(f0);
|
||||
var.assign(f1);
|
||||
kind.assign(f2);
|
||||
return true;
|
||||
}
|
||||
|
||||
static std::string ReadFileOrEmpty(const std::string &fn)
|
||||
{
|
||||
std::ifstream file(fn);
|
||||
if (!file.is_open())
|
||||
{
|
||||
std::cerr << "could not open file " << fn << "\n";
|
||||
return {};
|
||||
}
|
||||
std::stringstream buffer;
|
||||
buffer << file.rdbuf();
|
||||
return buffer.str();
|
||||
}
|
||||
|
||||
static std::vector<std::string> ExtractJitVarNames(const std::string
|
||||
&kernel_code)
|
||||
{
|
||||
std::stringstream ss(kernel_code);
|
||||
std::string line;
|
||||
std::vector<std::string> var_names;
|
||||
std::unordered_set<std::string> seen_vars;
|
||||
|
||||
while (std::getline(ss, line))
|
||||
{
|
||||
std::string type, var, kind;
|
||||
if (ParseJitDirective(line, type, var, kind))
|
||||
{
|
||||
MFEM_VERIFY(IsValidIdentifier(var),
|
||||
"$JIT variable must be a valid identifier: " << var);
|
||||
MFEM_VERIFY(seen_vars.insert(var).second,
|
||||
"duplicate $JIT variable name: " << var);
|
||||
var_names.push_back(var);
|
||||
}
|
||||
}
|
||||
return var_names;
|
||||
}
|
||||
|
||||
static std::string RewriteKernelForJit(std::string kernel_code,
|
||||
const std::vector<std::string> &jit_values)
|
||||
{
|
||||
std::stringstream ss(kernel_code);
|
||||
std::string line;
|
||||
|
||||
std::string out;
|
||||
out.reserve(kernel_code.size() + 128);
|
||||
|
||||
bool have_pending = false;
|
||||
size_t pending_index = 0;
|
||||
std::string pending_type;
|
||||
std::string pending_var;
|
||||
std::unordered_set<std::string> seen_vars;
|
||||
|
||||
while (std::getline(ss, line))
|
||||
{
|
||||
line.push_back('\n');
|
||||
|
||||
if (have_pending)
|
||||
{
|
||||
MFEM_VERIFY(pending_index < jit_values.size(),
|
||||
"not enough JIT values provided");
|
||||
const size_t indent_end = line.find_first_not_of(" \t");
|
||||
const std::string indent =
|
||||
(indent_end == std::string::npos) ? std::string() :
|
||||
line.substr(0, indent_end);
|
||||
out += indent + "const " + pending_type + " " + pending_var + " = " +
|
||||
jit_values[pending_index] + ";\n";
|
||||
have_pending = false;
|
||||
++pending_index;
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string type, var, kind;
|
||||
if (ParseJitDirective(line, type, var, kind))
|
||||
{
|
||||
MFEM_VERIFY(IsValidIdentifier(var),
|
||||
"$JIT variable must be a valid identifier: " << var);
|
||||
MFEM_VERIFY(kind == "generic",
|
||||
"unsupported $JIT kind: " << kind);
|
||||
MFEM_VERIFY(seen_vars.insert(var).second,
|
||||
"duplicate $JIT variable name: " << var);
|
||||
|
||||
pending_type = std::move(type);
|
||||
pending_var = std::move(var);
|
||||
have_pending = true;
|
||||
continue; // drop directive line
|
||||
}
|
||||
|
||||
out += line;
|
||||
}
|
||||
|
||||
MFEM_VERIFY(!have_pending,
|
||||
"$JIT directive must annotate a following line");
|
||||
MFEM_VERIFY(jit_values.size() == pending_index,
|
||||
"JIT value count must match number of $JIT directives");
|
||||
return out;
|
||||
}
|
||||
|
||||
static std::string GeneratedOutputPath(std::string_view original_path)
|
||||
{
|
||||
const size_t last_sep = original_path.find_last_of("/\\");
|
||||
const size_t dot = original_path.find_last_of('.');
|
||||
const bool dot_in_filename =
|
||||
(dot != std::string_view::npos) &&
|
||||
(last_sep == std::string_view::npos || dot > last_sep);
|
||||
|
||||
const std::string_view base =
|
||||
dot_in_filename ? original_path.substr(0, dot) : original_path;
|
||||
return std::string(base) + "_generated.hpp";
|
||||
}
|
||||
|
||||
static void WriteFileOrWarn(const std::string &path,
|
||||
const std::string &contents)
|
||||
{
|
||||
std::ofstream out(path);
|
||||
if (!out.is_open())
|
||||
{
|
||||
std::cerr << "could not write generated file " << path << "\n";
|
||||
return;
|
||||
}
|
||||
out << contents;
|
||||
}
|
||||
|
||||
class JitQFunction
|
||||
{
|
||||
public:
|
||||
template <typename ImplT, size_t N>
|
||||
JitQFunction(ImplT, const std::string &fn,
|
||||
const std::array<bool, N> &activity_map)
|
||||
{
|
||||
using qf_signature = typename
|
||||
mfem::future::get_function_signature<
|
||||
decltype(&ImplT::operator())>::type;
|
||||
using qf_param_ts = typename qf_signature::parameter_ts;
|
||||
constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
|
||||
static_assert(N == nparams, "activity_map size must match qfunc arity");
|
||||
|
||||
this->fn = fn;
|
||||
this->nparams = nparams;
|
||||
this->activity_map.reserve(N);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
this->activity_map.push_back(activity_map[i]);
|
||||
}
|
||||
{
|
||||
const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
|
||||
this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
|
||||
}
|
||||
this->return_type = TypeNameString<typename qf_signature::return_t>();
|
||||
this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
|
||||
this->impl_type_name = TypeNameString<ImplT>();
|
||||
this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
|
||||
}
|
||||
|
||||
template <typename ReturnT, typename... Args>
|
||||
ReturnT run(std::string_view name,
|
||||
std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
|
||||
Args&&... args)
|
||||
{
|
||||
auto ordered_values = MatchJitValues(jit_values);
|
||||
auto &mod = GetOrCreateModule(ordered_values);
|
||||
auto &instance = mod.instantiate(std::string(name), std::string());
|
||||
return instance.template run<ReturnT>(std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename ReturnT, typename... Args>
|
||||
ReturnT run_primal(
|
||||
std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
|
||||
Args&&... args)
|
||||
{
|
||||
return run<ReturnT>(qfunc_name, jit_values,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename ReturnT, typename... Args>
|
||||
ReturnT run_derivative(
|
||||
std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
|
||||
Args&&... args)
|
||||
{
|
||||
return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<std::string_view> MatchJitValues(
|
||||
std::initializer_list<std::pair<std::string_view, std::string_view>>
|
||||
named_values) const
|
||||
{
|
||||
std::unordered_map<std::string_view, std::string_view> value_map;
|
||||
for (const auto &[name, value] : named_values)
|
||||
{
|
||||
value_map[name] = value;
|
||||
}
|
||||
|
||||
std::vector<std::string_view> ordered_values;
|
||||
ordered_values.reserve(jit_var_names.size());
|
||||
for (const auto &var_name : jit_var_names)
|
||||
{
|
||||
auto it = value_map.find(var_name);
|
||||
MFEM_VERIFY(it != value_map.end(),
|
||||
"missing JIT value for variable: " << var_name);
|
||||
ordered_values.push_back(it->second);
|
||||
}
|
||||
|
||||
MFEM_VERIFY(ordered_values.size() == named_values.size(),
|
||||
"provided " << named_values.size() << " JIT values but expected "
|
||||
<< jit_var_names.size());
|
||||
return ordered_values;
|
||||
}
|
||||
|
||||
|
||||
std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
|
||||
{
|
||||
std::string module_code =
|
||||
RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
|
||||
module_code += "\n\n";
|
||||
module_code += "// --- generated ---\n";
|
||||
module_code +=
|
||||
"template <typename return_type, typename... Args>\n"
|
||||
"return_type __enzyme_fwddiff(Args...);\n"
|
||||
"\n"
|
||||
"extern int enzyme_const;\n"
|
||||
"extern int enzyme_dup;\n"
|
||||
"\n";
|
||||
|
||||
// Generate a primal wrapper with the requested symbol name, so the kernel
|
||||
// header can just define the qfunc as a functor.
|
||||
//
|
||||
// Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
|
||||
// when there are no user template args, so keep the wrapper itself a
|
||||
// template (with a default parameter) while still doing literal `$JIT`
|
||||
// replacements in the kernel code.
|
||||
module_code += "template <typename = void>\n";
|
||||
module_code += return_type + " " +
|
||||
std::string(qfunc_name) + "(";
|
||||
bool first = true;
|
||||
for (size_t i = 0; i < nparams; ++i)
|
||||
{
|
||||
if (!first) { module_code += ", "; }
|
||||
first = false;
|
||||
module_code += param_types[i] + " Arg" + std::to_string(i);
|
||||
}
|
||||
module_code += ")\n";
|
||||
module_code += "{\n";
|
||||
module_code += " " + impl_type_name + " qf;\n";
|
||||
if (return_is_void)
|
||||
{
|
||||
module_code += " ";
|
||||
}
|
||||
else
|
||||
{
|
||||
module_code += " return ";
|
||||
}
|
||||
module_code += "qf(";
|
||||
for (size_t i = 0; i < nparams; ++i)
|
||||
{
|
||||
if (i) { module_code += ", "; }
|
||||
module_code += "Arg" + std::to_string(i);
|
||||
}
|
||||
module_code += ");\n";
|
||||
module_code += "}\n\n";
|
||||
|
||||
module_code += "template <typename = void>\n";
|
||||
module_code += return_type + " " +
|
||||
std::string(qfunc_name) + "_fwddiff(";
|
||||
|
||||
first = true;
|
||||
for (size_t i = 0; i < nparams; ++i)
|
||||
{
|
||||
if (!first) { module_code += ", "; }
|
||||
first = false;
|
||||
module_code += param_types[i] + " Arg" + std::to_string(i);
|
||||
if (activity_map[i])
|
||||
{
|
||||
module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
|
||||
}
|
||||
}
|
||||
module_code += ")\n";
|
||||
module_code += "{\n";
|
||||
if (return_is_void)
|
||||
{
|
||||
module_code += " __enzyme_fwddiff<void>(\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
module_code += " return __enzyme_fwddiff<" +
|
||||
return_type + ">(\n";
|
||||
}
|
||||
module_code += " (void*)" + std::string(qfunc_name) + "<>";
|
||||
module_code += ",\n";
|
||||
for (size_t i = 0; i < nparams; ++i)
|
||||
{
|
||||
if (activity_map[i])
|
||||
{
|
||||
module_code += " enzyme_dup, Arg" + std::to_string(i) +
|
||||
", dArg" + std::to_string(i);
|
||||
}
|
||||
else
|
||||
{
|
||||
module_code += " enzyme_const, Arg" + std::to_string(i);
|
||||
}
|
||||
module_code += (i + 1 == nparams) ? ");\n" : ",\n";
|
||||
}
|
||||
module_code += "}\n";
|
||||
|
||||
WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
|
||||
return module_code;
|
||||
}
|
||||
|
||||
proteus::CppJitModule &GetOrCreateModule(
|
||||
const std::vector<std::string_view> &jit_values)
|
||||
{
|
||||
std::string key;
|
||||
for (const auto &val : jit_values)
|
||||
{
|
||||
if (!key.empty()) { key += ","; }
|
||||
key += val;
|
||||
}
|
||||
|
||||
auto it = modules.find(key);
|
||||
if (it != modules.end())
|
||||
{
|
||||
return *it->second;
|
||||
}
|
||||
|
||||
std::vector<std::string> values(jit_values.begin(), jit_values.end());
|
||||
std::string code = BuildModuleCode(values);
|
||||
auto mod = std::make_unique<proteus::CppJitModule>("host", code,
|
||||
DefaultExtraArgs());
|
||||
auto [inserted, ok] = modules.emplace(key, std::move(mod));
|
||||
MFEM_VERIFY(ok, "failed to cache JIT module");
|
||||
return *inserted->second;
|
||||
}
|
||||
|
||||
static std::vector<std::string> DefaultExtraArgs()
|
||||
{
|
||||
return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
|
||||
}
|
||||
|
||||
std::string qfunc_name = "qfunc_wrapper";
|
||||
std::string fn;
|
||||
size_t nparams = 0;
|
||||
std::vector<bool> activity_map;
|
||||
std::vector<std::string> param_types;
|
||||
std::string return_type;
|
||||
bool return_is_void = false;
|
||||
std::string impl_type_name;
|
||||
std::vector<std::string> jit_var_names;
|
||||
std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
|
||||
};
|
||||
|
||||
int main()
|
||||
{
|
||||
const size_t N = 4;
|
||||
const size_t M = 5;
|
||||
const double A = 123.4;
|
||||
|
||||
std::vector<double> X(N);
|
||||
std::vector<double> Y(N);
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
X[i] = static_cast<double>(i + 1);
|
||||
Y[i] = static_cast<double>(N - i);
|
||||
}
|
||||
|
||||
// // >>> user interface calls
|
||||
// const std::string kernel_path = std::string(util::thisFileDir) +
|
||||
// "/jitplayground.hpp";
|
||||
// JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
|
||||
// // <<< user interface calls
|
||||
|
||||
// // this will happen internally in dFEM
|
||||
|
||||
daxpy_op op;
|
||||
printf("\n\nfunction call\n");
|
||||
op(&A, X.data(), Y.data(), &N);
|
||||
|
||||
// reset X for the derivative test
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
X[i] = static_cast<double>(i + 1);
|
||||
Y[i] = static_cast<double>(N - i);
|
||||
}
|
||||
|
||||
std::vector<double> dX(N, 1.0);
|
||||
printf("\n\nforward diff call\n");
|
||||
daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
|
||||
|
||||
std::vector<double> dX_manual(N, A);
|
||||
|
||||
printf("\n\nderivative checks\n");
|
||||
std::cout << "dX: ";
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
|
||||
}
|
||||
|
||||
std::cout << "dX_manual: ";
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
|
||||
}
|
||||
|
||||
double max_abs_err = 0.0;
|
||||
for (size_t i = 0; i < N; ++i)
|
||||
{
|
||||
max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
|
||||
}
|
||||
std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int main()
|
||||
{
|
||||
std::cerr << "Proteus is required to run this example.\n";
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
#endif // MFEM_USE_PROTEUS
|
||||
@@ -1,58 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <vector>
|
||||
#include <type_traits>
|
||||
|
||||
#include "proteus/JitInterface.h"
|
||||
|
||||
struct daxpy_op
|
||||
{
|
||||
void operator()(
|
||||
const double *a,
|
||||
double *x,
|
||||
const double *y,
|
||||
const size_t *N) const
|
||||
{
|
||||
const size_t n = *N;
|
||||
auto lam = [=, n = proteus::jit_variable(n)]
|
||||
() __attribute__((annotate("jit")))
|
||||
{
|
||||
printf("N = %zu\n", n);
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
|
||||
x[i] = *a * x[i] + y[i];
|
||||
printf("updated x[%zu] = %f\n", i, x[i]);
|
||||
}
|
||||
};
|
||||
|
||||
proteus::register_lambda(lam);
|
||||
|
||||
lam();
|
||||
}
|
||||
};
|
||||
|
||||
template <typename return_type, typename... Args>
|
||||
return_type __enzyme_fwddiff(Args...);
|
||||
|
||||
extern int enzyme_const;
|
||||
extern int enzyme_dup;
|
||||
|
||||
void daxpy_op_wrapper(const double * Arg0, double * Arg1,
|
||||
const double * Arg2, const size_t *Arg3)
|
||||
{
|
||||
daxpy_op qf;
|
||||
qf(Arg0, Arg1, Arg2, Arg3);
|
||||
}
|
||||
|
||||
void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
|
||||
double * dArg1, const double * Arg2, const size_t *Arg3)
|
||||
{
|
||||
__enzyme_fwddiff<void>(
|
||||
(void*)daxpy_op_wrapper,
|
||||
enzyme_const, Arg0,
|
||||
enzyme_dup, Arg1, dArg1,
|
||||
enzyme_const, Arg2,
|
||||
enzyme_const, Arg3);
|
||||
}
|
||||
+5
-5
@@ -31,6 +31,9 @@ SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
|
||||
PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
|
||||
ex22p ex24p ex25p ex26p ex34p ex35p
|
||||
|
||||
ifeq ($(MFEM_USE_ARPACK),YES)
|
||||
SEQ_EXAMPLES += ex11 ex13
|
||||
endif
|
||||
ifeq ($(MFEM_USE_LAPACK),YES)
|
||||
SEQ_EXAMPLES += ex38
|
||||
endif
|
||||
@@ -71,7 +74,6 @@ endif
|
||||
|
||||
SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
|
||||
SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
|
||||
SUBDIRS_TEST_NOCLEAN = $(addsuffix /test-noclean,$(SUBDIRS))
|
||||
SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))
|
||||
SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
|
||||
|
||||
@@ -88,9 +90,8 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
|
||||
|
||||
all: $(EXAMPLES) $(SUBDIRS_ALL)
|
||||
|
||||
.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) \
|
||||
$(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
|
||||
$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) $(SUBDIRS_CLEAN):
|
||||
.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
|
||||
$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN):
|
||||
$(MAKE) -C $(@D) $(@F)
|
||||
$(SUBDIRS_TPRINT):
|
||||
@$(MAKE) -C $(@D) $(@F)
|
||||
@@ -109,7 +110,6 @@ endif
|
||||
MFEM_TESTS = EXAMPLES
|
||||
include $(MFEM_TEST_MK)
|
||||
test: $(SUBDIRS_TEST)
|
||||
test-noclean: $(SUBDIRS_TEST_NOCLEAN)
|
||||
test-print: $(SUBDIRS_TPRINT)
|
||||
|
||||
# Testing: Parallel vs. serial runs
|
||||
|
||||
+16
-56
@@ -57,8 +57,6 @@ set(SRCS
|
||||
integ/lininteg_domain_grad.cpp
|
||||
integ/lininteg_domain_vectorfe.cpp
|
||||
integ/nonlininteg_vecconvection_pa.cpp
|
||||
integ/nonlininteg_vecconvection_pa_diag.cpp
|
||||
integ/nonlininteg_vecconvection_pa_grad.cpp
|
||||
integ/nonlininteg_vecconvection_mf.cpp
|
||||
coefficient.cpp
|
||||
complex_fem.cpp
|
||||
@@ -67,7 +65,6 @@ set(SRCS
|
||||
dgmassinv.cpp
|
||||
doftrans.cpp
|
||||
dfem/doperator.cpp
|
||||
dfem/backends/local_qf/kernels.cpp
|
||||
eltrans.cpp
|
||||
batchitrans.cpp
|
||||
estimators.cpp
|
||||
@@ -124,11 +121,6 @@ set(SRCS
|
||||
qinterp/eval_hdiv.cpp
|
||||
qinterp/grad_by_nodes.cpp
|
||||
qinterp/grad_by_vdim.cpp
|
||||
qinterp/grad_transpose.cpp
|
||||
qinterp/grad_transpose_by_nodes.cpp
|
||||
qinterp/grad_transpose_by_vdim.cpp
|
||||
qinterp/eval_transpose.cpp
|
||||
qinterp/eval_transpose_by_vdim.cpp
|
||||
qspace.cpp
|
||||
quadinterpolator.cpp
|
||||
quadinterpolator_face.cpp
|
||||
@@ -141,7 +133,7 @@ set(SRCS
|
||||
tmop/assemble/diag2.cpp
|
||||
tmop/assemble/grad2_limit.cpp
|
||||
tmop/assemble/grad2.cpp
|
||||
tmop/assemble/diag3_limit.cpp
|
||||
tmop/assemble/diag3_limit.cpp
|
||||
tmop/assemble/diag3.cpp
|
||||
tmop/assemble/grad3_limit.cpp
|
||||
tmop/assemble/grad3.cpp
|
||||
@@ -179,12 +171,8 @@ set(SRCS
|
||||
tmop_tools.cpp
|
||||
tmop_amr.cpp
|
||||
gslib.cpp
|
||||
gslib/findptsedge_local_2.cpp
|
||||
gslib/findptsedge_local_3.cpp
|
||||
gslib/findptssurf_local_3.cpp
|
||||
gslib/findpts_local_2.cpp
|
||||
gslib/findpts_local_3.cpp
|
||||
gslib/interpolate_local_1.cpp
|
||||
gslib/interpolate_local_2.cpp
|
||||
gslib/interpolate_local_3.cpp
|
||||
transfer.cpp
|
||||
@@ -203,20 +191,14 @@ set(HDRS
|
||||
integ/bilininteg_dgtrace_kernels.hpp
|
||||
integ/bilininteg_vecdiffusion_kernels.hpp
|
||||
integ/bilininteg_convection_kernels.hpp
|
||||
integ/bilininteg_diffusion_pa_simplices.hpp
|
||||
integ/bilininteg_diffusion_kernels.hpp
|
||||
integ/bilininteg_elasticity_kernels.hpp
|
||||
integ/bilininteg_hcurl_kernels.hpp
|
||||
integ/bilininteg_hdiv_kernels.hpp
|
||||
integ/bilininteg_hcurlhdiv_kernels.hpp
|
||||
integ/bilininteg_mass_kernels.hpp
|
||||
integ/bilininteg_mass_pa_simplices.hpp
|
||||
integ/bilininteg_vecdiffusion_pa.hpp
|
||||
integ/bilininteg_vecdiv_pa.hpp
|
||||
integ/bilininteg_vecmass_pa.hpp
|
||||
integ/nonlininteg_vecconvection_pa.hpp
|
||||
integ/nonlininteg_vecconvection_pa_diag.hpp
|
||||
integ/nonlininteg_vecconvection_pa_grad.hpp
|
||||
coefficient.hpp
|
||||
complex_fem.hpp
|
||||
convergence.hpp
|
||||
@@ -225,32 +207,13 @@ set(HDRS
|
||||
dgmassinv_kernels.hpp
|
||||
doftrans.hpp
|
||||
dfem/doperator.hpp
|
||||
dfem/fielddescriptor.hpp
|
||||
dfem/fieldoperator.hpp
|
||||
dfem/integrator_ctx.hpp
|
||||
dfem/integrate.hpp
|
||||
dfem/parameterspace.hpp
|
||||
dfem/tensor_functions.hpp
|
||||
dfem/qfunction_apply.hpp
|
||||
dfem/qfunction_transform.hpp
|
||||
dfem/tuple.hpp
|
||||
dfem/util.hpp
|
||||
dfem/backends/util.hpp
|
||||
dfem/backends/global_qf/action.hpp
|
||||
dfem/backends/global_qf/derivative_action.hpp
|
||||
dfem/backends/global_qf/derivative_apply.hpp
|
||||
dfem/backends/global_qf/derivative_apply_transpose.hpp
|
||||
dfem/backends/global_qf/derivative_setup.hpp
|
||||
dfem/backends/global_qf/prelude.hpp
|
||||
dfem/backends/local_qf/action.hpp
|
||||
dfem/backends/local_qf/derivative_action.hpp
|
||||
dfem/backends/local_qf/derivative_apply.hpp
|
||||
dfem/backends/local_qf/derivative_apply_transpose.hpp
|
||||
dfem/backends/local_qf/derivative_assemble.hpp
|
||||
dfem/backends/local_qf/derivative_assemble_diagonal.hpp
|
||||
dfem/backends/local_qf/derivative_setup.hpp
|
||||
dfem/backends/local_qf/kernels.hpp
|
||||
dfem/backends/local_qf/kernels_ho.hpp
|
||||
dfem/backends/local_qf/kernels_lo.hpp
|
||||
dfem/backends/local_qf/prelude.hpp
|
||||
dfem/backends/local_qf/util.hpp
|
||||
eltrans.hpp
|
||||
estimators.hpp
|
||||
fe.hpp
|
||||
@@ -315,10 +278,8 @@ set(HDRS
|
||||
qfunction.hpp
|
||||
qinterp/det.hpp
|
||||
qinterp/eval.hpp
|
||||
qinterp/eval_transpose.hpp
|
||||
qinterp/eval_hdiv.hpp
|
||||
qinterp/grad.hpp
|
||||
qinterp/grad_transpose.hpp
|
||||
qspace.hpp
|
||||
quadinterpolator.hpp
|
||||
quadinterpolator_face.hpp
|
||||
@@ -344,7 +305,6 @@ set(HDRS
|
||||
tmop_tools.hpp
|
||||
tmop_amr.hpp
|
||||
gslib.hpp
|
||||
gslib/gslib_kernel_helpers.hpp
|
||||
transfer.hpp
|
||||
hyperbolic.hpp
|
||||
integrator.hpp
|
||||
@@ -353,36 +313,36 @@ set(HDRS
|
||||
)
|
||||
|
||||
if (MFEM_USE_SIDRE)
|
||||
list(APPEND SRCS sidredatacollection.cpp)
|
||||
list(APPEND HDRS sidredatacollection.hpp)
|
||||
list(APPEND SRCS sidredatacollection.cpp)
|
||||
list(APPEND HDRS sidredatacollection.hpp)
|
||||
endif()
|
||||
|
||||
if (MFEM_USE_CONDUIT)
|
||||
list(APPEND SRCS conduitdatacollection.cpp)
|
||||
list(APPEND HDRS conduitdatacollection.hpp)
|
||||
list(APPEND SRCS conduitdatacollection.cpp)
|
||||
list(APPEND HDRS conduitdatacollection.hpp)
|
||||
endif()
|
||||
|
||||
if (MFEM_USE_ADIOS2)
|
||||
list(APPEND SRCS adios2datacollection.cpp)
|
||||
list(APPEND HDRS adios2datacollection.hpp)
|
||||
list(APPEND SRCS adios2datacollection.cpp)
|
||||
list(APPEND HDRS adios2datacollection.hpp)
|
||||
endif()
|
||||
|
||||
if (MFEM_USE_FMS)
|
||||
list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
|
||||
list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
|
||||
list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
|
||||
list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
|
||||
endif()
|
||||
|
||||
if (MFEM_USE_MPI)
|
||||
list(APPEND SRCS
|
||||
list(APPEND SRCS
|
||||
pbilinearform.cpp
|
||||
pfespace.cpp
|
||||
pgridfunc.cpp
|
||||
plinearform.cpp
|
||||
pnonlinearform.cpp
|
||||
prestriction.cpp)
|
||||
# If this list (HDRS -> HEADERS) is used for install, we probably want the
|
||||
# headers added all the time.
|
||||
list(APPEND HDRS
|
||||
# If this list (HDRS -> HEADERS) is used for install, we probably want the
|
||||
# headers added all the time.
|
||||
list(APPEND HDRS
|
||||
pbilinearform.hpp
|
||||
pfespace.hpp
|
||||
pgridfunc.hpp
|
||||
|
||||
+4
-22
@@ -1345,8 +1345,7 @@ real_t DiffusionIntegrator::ComputeFluxEnergy
|
||||
}
|
||||
|
||||
const IntegrationRule &DiffusionIntegrator::GetRule(
|
||||
const FiniteElement &trial_fe, const FiniteElement &test_fe,
|
||||
const bool stroud)
|
||||
const FiniteElement &trial_fe, const FiniteElement &test_fe)
|
||||
{
|
||||
int order;
|
||||
if (trial_fe.Space() == FunctionSpace::Pk)
|
||||
@@ -1363,15 +1362,7 @@ const IntegrationRule &DiffusionIntegrator::GetRule(
|
||||
{
|
||||
return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
|
||||
if (stroud)
|
||||
{
|
||||
return StroudIntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
else
|
||||
{
|
||||
return IntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
return IntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
|
||||
MassIntegrator::MassIntegrator(const IntegrationRule *ir)
|
||||
@@ -1458,8 +1449,7 @@ void MassIntegrator::AssembleElementMatrix2(
|
||||
|
||||
const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
|
||||
const FiniteElement &test_fe,
|
||||
const ElementTransformation &Trans,
|
||||
const bool stroud)
|
||||
const ElementTransformation &Trans)
|
||||
{
|
||||
// int order = trial_fe.GetOrder() + test_fe.GetOrder();
|
||||
const int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
|
||||
@@ -1468,15 +1458,7 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
|
||||
{
|
||||
return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
|
||||
if (stroud)
|
||||
{
|
||||
return StroudIntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
else
|
||||
{
|
||||
return IntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
return IntRules.Get(trial_fe.GetGeomType(), order);
|
||||
}
|
||||
|
||||
|
||||
|
||||
+7
-94
@@ -2184,22 +2184,11 @@ public:
|
||||
const Vector&, const Vector&,
|
||||
Vector&, const int, const int);
|
||||
|
||||
using ApplySimplexKernelType = void(*)(const int, const bool, const Array<int>&,
|
||||
const Array<int>&,
|
||||
const Array<int>&, const Array<int>&, const Array<int>&,
|
||||
const Array<real_t>&, const Array<real_t>&,
|
||||
const Array<real_t>&, const Array<real_t>&,
|
||||
const Array<real_t>&, const Array<real_t>&,
|
||||
const Vector&, const Vector&,
|
||||
Vector&, const int, const int);
|
||||
|
||||
using DiagonalKernelType = void(*)(const int, const bool, const Array<real_t>&,
|
||||
const Array<real_t>&, const Vector&, Vector&,
|
||||
const int, const int);
|
||||
|
||||
MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
|
||||
MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
|
||||
int));
|
||||
MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
|
||||
struct Kernels { Kernels(); };
|
||||
|
||||
@@ -2352,8 +2341,7 @@ public:
|
||||
void AddMultPatchPA(const int patch, const Vector &x, Vector &y) const;
|
||||
|
||||
static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
|
||||
const FiniteElement &test_fe,
|
||||
const bool stroud = false);
|
||||
const FiniteElement &test_fe);
|
||||
|
||||
bool SupportsCeed() const override { return DeviceCanUseCeed(); }
|
||||
|
||||
@@ -2364,13 +2352,6 @@ public:
|
||||
{
|
||||
ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
|
||||
DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
|
||||
AddSimplexSpecialization<DIM,D1D,Q1D>();
|
||||
}
|
||||
|
||||
template <int DIM, int D1D, int Q1D>
|
||||
static void AddSimplexSpecialization()
|
||||
{
|
||||
ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
|
||||
}
|
||||
protected:
|
||||
const IntegrationRule* GetDefaultIntegrationRule(
|
||||
@@ -2407,22 +2388,11 @@ public:
|
||||
const Array<real_t>&, const Vector&,
|
||||
const Vector&, Vector&, const int, const int);
|
||||
|
||||
using ApplySimplexKernelType = void(*)(const int, const Array<int>&,
|
||||
const Array<int>&,
|
||||
const Array<int>&, const Array<int>&, const Array<int>&,
|
||||
const Array<real_t>&, const Array<real_t>&,
|
||||
const Array<real_t>&, const Array<real_t>&,
|
||||
const Array<real_t>&, const Array<real_t>&,
|
||||
const Vector&, const Vector&, Vector&,
|
||||
const int, const int);
|
||||
|
||||
using DiagonalKernelType = void(*)(const int, const Array<real_t>&,
|
||||
const Vector&, Vector&, const int,
|
||||
const int);
|
||||
|
||||
MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
|
||||
MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
|
||||
int));
|
||||
MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
|
||||
struct Kernels { Kernels(); };
|
||||
|
||||
@@ -2471,8 +2441,7 @@ public:
|
||||
|
||||
static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
|
||||
const FiniteElement &test_fe,
|
||||
const ElementTransformation &Trans,
|
||||
const bool stroud = false);
|
||||
const ElementTransformation &Trans);
|
||||
|
||||
bool SupportsCeed() const override { return DeviceCanUseCeed(); }
|
||||
|
||||
@@ -2483,13 +2452,6 @@ public:
|
||||
{
|
||||
ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
|
||||
DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
|
||||
AddSimplexSpecialization<DIM,D1D,Q1D>();
|
||||
}
|
||||
|
||||
template <int DIM, int D1D, int Q1D>
|
||||
static void AddSimplexSpecialization()
|
||||
{
|
||||
ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
|
||||
}
|
||||
|
||||
protected:
|
||||
@@ -2689,22 +2651,14 @@ public:
|
||||
void AddMultMF(const Vector &x, Vector &y) const override;
|
||||
bool SupportsCeed() const override { return DeviceCanUseCeed(); }
|
||||
|
||||
// PA AddMultPA kernels
|
||||
using VectorMassAddMultPAType =
|
||||
void(*)(const int, const int,
|
||||
const Array<real_t>&, const Vector&,
|
||||
const Vector&, Vector&, const int, const int);
|
||||
|
||||
MFEM_REGISTER_KERNELS(VectorMassAddMultPA,
|
||||
VectorMassAddMultPAType,
|
||||
(int, int, int));
|
||||
|
||||
// PA DiagonalPA kernels
|
||||
using VectorMassAssembleDiagonalPAType =
|
||||
void(*)(const int, const int, const int,
|
||||
const real_t*, const real_t*, real_t*);
|
||||
MFEM_REGISTER_KERNELS(VectorMassAssembleDiagonalPA,
|
||||
VectorMassAssembleDiagonalPAType,
|
||||
(int /*dim*/, int /*q1d*/));
|
||||
};
|
||||
|
||||
|
||||
@@ -3003,10 +2957,11 @@ public:
|
||||
vector (diagonal matrix), or matrix), trial function $u$ is in $H(curl$ or
|
||||
$H(div)$, and test function $v$ is in $H(curl$, $H(div)$, or $v=(v_1,\dots,v_n)$, where
|
||||
$v_i$ are in $H^1$. */
|
||||
class VectorFEMassIntegrator : public BilinearFormIntegrator
|
||||
class VectorFEMassIntegrator: public BilinearFormIntegrator
|
||||
{
|
||||
private:
|
||||
void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq);
|
||||
void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
|
||||
{ Q = q; DQ = dq; MQ = mq; }
|
||||
|
||||
#ifndef MFEM_THREAD_SAFE
|
||||
Vector shape;
|
||||
@@ -3029,8 +2984,7 @@ protected:
|
||||
const DofToQuad *mapsOtest; ///< Not owned. DOF-to-quad map, open.
|
||||
const DofToQuad *mapsCtest; ///< Not owned. DOF-to-quad map, closed.
|
||||
const GeometricFactors *geom; ///< Not owned
|
||||
int dim, ne, nq, dofs1D, dofs1Dtest, quad1D;
|
||||
FiniteElement::DerivType trial_fetype, test_fetype;
|
||||
int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
|
||||
bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
|
||||
|
||||
public:
|
||||
@@ -3061,29 +3015,6 @@ public:
|
||||
const bool add) override;
|
||||
|
||||
const Coefficient *GetCoefficient() const { return Q; }
|
||||
|
||||
using ApplyKernelType =
|
||||
void (*)(const int NE, bool symmetric, const bool scalar_coeff,
|
||||
const Array<real_t> &trialBO, const Array<real_t> &trialBC,
|
||||
const Array<real_t> &testBOt, const Array<real_t> &testBCt,
|
||||
const Vector &pa_data, const Vector &x, Vector &y,
|
||||
const int triald1d, const int testd1d, const int q1d);
|
||||
|
||||
/// parameters: trial_fetype, test_fetype, ndims, trial_d1d, test_d1d, q1d
|
||||
MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType,
|
||||
(FiniteElement::DerivType, FiniteElement::DerivType,
|
||||
int, int, int, int));
|
||||
|
||||
struct Kernels { Kernels(); };
|
||||
|
||||
template <FiniteElement::DerivType TrialType,
|
||||
FiniteElement::DerivType TestType, int DIM, int TRIAL_D1D,
|
||||
int TEST_D1D, int Q1D>
|
||||
static void AddSpecialization()
|
||||
{
|
||||
ApplyPAKernels::Specialization<TrialType, TestType, DIM, TRIAL_D1D,
|
||||
TEST_D1D, Q1D>::Add();
|
||||
}
|
||||
};
|
||||
|
||||
/** Integrator for $(Q \nabla \cdot u, v)$ where $u=(u_1,\cdots,u_n)$ and all $u_i$ are in the same
|
||||
@@ -3129,24 +3060,6 @@ public:
|
||||
void AddMultPA(const Vector &x, Vector &y) const override;
|
||||
void AddMultTransposePA(const Vector &x, Vector &y) const override;
|
||||
|
||||
using VectorDivergenceAddMultPAType =
|
||||
void (*)(const int ne,
|
||||
const Array<real_t> &b, const Array<real_t> &g, const Array<real_t> &bt,
|
||||
const Vector &op, const Vector &x, Vector &y,
|
||||
const int tr_d1d, const int te_d1d, const int q1d);
|
||||
MFEM_REGISTER_KERNELS(VectorDivergenceAddMultPA,
|
||||
VectorDivergenceAddMultPAType,
|
||||
(int, int, int, int));
|
||||
|
||||
using VectorDivergenceAddMultTransposePAType =
|
||||
void (*)(const int ne,
|
||||
const Array<real_t> &bt, const Array<real_t> >, const Array<real_t> &b,
|
||||
const Vector &q, const Vector &x, Vector &y,
|
||||
const int tr_d1d, const int te_d1d, const int q1d);
|
||||
MFEM_REGISTER_KERNELS(VectorDivergenceAddMultTransposePA,
|
||||
VectorDivergenceAddMultTransposePAType,
|
||||
(int, int, int, int));
|
||||
|
||||
static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
|
||||
const FiniteElement &test_fe,
|
||||
const ElementTransformation &Trans);
|
||||
|
||||
@@ -54,8 +54,6 @@ void Coefficient::Project(QuadratureFunction &qf)
|
||||
QuadratureSpaceBase &qspace = *qf.GetSpace();
|
||||
const int ne = qspace.GetNE();
|
||||
Vector values;
|
||||
// GetValues makes a reference, but we need it to be valid on Host
|
||||
qf.HostWrite();
|
||||
for (int iel = 0; iel < ne; ++iel)
|
||||
{
|
||||
qf.GetValues(iel, values);
|
||||
@@ -329,8 +327,6 @@ void VectorCoefficient::Project(QuadratureFunction &qf)
|
||||
const int ne = qspace.GetNE();
|
||||
DenseMatrix values;
|
||||
Vector col;
|
||||
// GetValues makes a reference, but we need it to be valid on Host
|
||||
qf.HostWrite();
|
||||
for (int iel = 0; iel < ne; ++iel)
|
||||
{
|
||||
qf.GetValues(iel, values);
|
||||
@@ -699,8 +695,6 @@ void MatrixCoefficient::Project(QuadratureFunction &qf, bool transpose)
|
||||
QuadratureSpaceBase &qspace = *qf.GetSpace();
|
||||
const int ne = qspace.GetNE();
|
||||
DenseMatrix values, matrix;
|
||||
// GetValues makes a reference, but we need it to be valid on Host
|
||||
qf.HostWrite();
|
||||
for (int iel = 0; iel < ne; ++iel)
|
||||
{
|
||||
qf.GetValues(iel, values);
|
||||
|
||||
+34
-944
File diff suppressed because it is too large
Load Diff
@@ -166,75 +166,6 @@ public:
|
||||
return sqrt(err_r * err_r + err_i * err_i);
|
||||
}
|
||||
|
||||
/// @brief Returns Max|u_ex - u_h| error for complex-valued H1 or L2 elements
|
||||
///
|
||||
/// Compute the $L_\infty$ error across the entire domain.
|
||||
///
|
||||
/// @param[in] exsolr Coefficient object reproducing the real part of the
|
||||
/// anticipated values of the scalar field, Re(u_ex).
|
||||
/// @param[in] exsoli Coefficient object reproducing the imaginary part of
|
||||
/// the anticipated values of the scalar field, Im(u_ex).
|
||||
/// @param[in] irs Optional pointer to an array of custom integration
|
||||
/// rules e.g. higher order than the default rules. If
|
||||
/// present the array will be indexed by
|
||||
/// Geometry::Type.
|
||||
///
|
||||
/// @note Uses ComputeLpError internally. See the ComputeLpError
|
||||
/// documentation for generalizations of this error computation.
|
||||
///
|
||||
/// @note If an array of integration rules is provided through @a irs, be
|
||||
/// sure to include valid rules for each element type that may occur
|
||||
/// in the list of elements.
|
||||
///
|
||||
virtual real_t ComputeMaxError(Coefficient &exsolr,
|
||||
Coefficient &exsoli,
|
||||
const IntegrationRule *irs[] = NULL) const
|
||||
{
|
||||
return ComputeLpError(infinity(), exsolr, exsoli, NULL, irs);
|
||||
}
|
||||
|
||||
/// @brief Returns ||u_ex - u_h||_Lp for complex-valued H1 or L2 elements
|
||||
///
|
||||
/// Computes:
|
||||
/// $$(\sum_{elems} \int_{elem} w \, |u_{ex} - u_h|^p)^{1/p}$$
|
||||
/// Where:
|
||||
/// $$|u_{ex} - u_h| = \sqrt{Re(u_{ex} - u_h)^2 + Im(u_{ex} - u_h)^2}$$
|
||||
///
|
||||
/// @param[in] p Real value indicating the exponent of the $L^p$ norm.
|
||||
/// To avoid domain errors p should have a positive value,
|
||||
/// either finite or infinite.
|
||||
/// @param[in] exsolr Coefficient object reproducing the real part of the
|
||||
/// anticipated values of the scalar field, Re(u_ex).
|
||||
/// @param[in] exsoli Coefficient object reproducing the imaginary part of
|
||||
/// the anticipated values of the scalar field, Im(u_ex).
|
||||
/// @param[in] weight Optional pointer to a Coefficient object reproducing
|
||||
/// a weighting function, w.
|
||||
/// @param[in] irs Optional pointer to an array of custom integration
|
||||
/// rules e.g. higher order than the default rules. If
|
||||
/// present the array will be indexed by Geometry::Type.
|
||||
/// @param[in] elems Optional pointer to a marker array, with a length
|
||||
/// equal to the number of local elements, indicating
|
||||
/// which elements to integrate over. Only those elements
|
||||
/// corresponding to non-zero entries in @a elems will
|
||||
/// contribute to the computed L2 error.
|
||||
///
|
||||
/// @note If an array of integration rules is provided through @a irs, be
|
||||
/// sure to include valid rules for each element type that may occur
|
||||
/// in the list of elements.
|
||||
///
|
||||
/// @note Quadratures with negative weights (as in some simplex integration
|
||||
/// rules in MFEM) can produce negative integrals even with
|
||||
/// non-negative integrands. To avoid returning negative errors this
|
||||
/// function uses the absolute values of the element-wise integrals.
|
||||
/// This may lead to results which are not entirely consistent with
|
||||
/// such integration rules.
|
||||
virtual real_t ComputeLpError(const real_t p,
|
||||
Coefficient &exsolr,
|
||||
Coefficient &exsoli,
|
||||
Coefficient *weight = NULL,
|
||||
const IntegrationRule *irs[] = NULL,
|
||||
const Array<int> *elems = NULL) const;
|
||||
|
||||
/// Save the ComplexGridFunction to an output stream.
|
||||
virtual void Save(std::ostream &out) const;
|
||||
|
||||
@@ -505,186 +436,6 @@ public:
|
||||
virtual ~SesquilinearForm();
|
||||
};
|
||||
|
||||
/** Class for a mixed sesquilinear form
|
||||
|
||||
A mixed sesquilinear form is a generalization of a mixed bilinear form to
|
||||
complex-valued fields. Mixed sesquilinear forms are linear in the second
|
||||
argument but the first argument involves a complex conjugate in the sense
|
||||
that:
|
||||
|
||||
a(alpha u, beta v) = conj(alpha) beta a(u, v)
|
||||
|
||||
The @a convention argument in the class's constructor is documented in the
|
||||
mfem::ComplexOperator class found in linalg/complex_operator.hpp.
|
||||
|
||||
When supplying integrators to the MixedSesquilinearForm either the real or
|
||||
imaginary integrator can be NULL. This indicates that the corresponding
|
||||
portion of the complex-valued material coefficient is equal to zero.
|
||||
*/
|
||||
class MixedSesquilinearForm
|
||||
{
|
||||
private:
|
||||
ComplexOperator::Convention conv;
|
||||
|
||||
MixedBilinearForm * mblfr;
|
||||
MixedBilinearForm * mblfi;
|
||||
|
||||
/* These methods check if the real/imag parts of the sesqulinear form are not
|
||||
empty */
|
||||
bool RealInteg();
|
||||
bool ImagInteg();
|
||||
|
||||
public:
|
||||
MixedSesquilinearForm(
|
||||
FiniteElementSpace * trial_fes,
|
||||
FiniteElementSpace * test_fes,
|
||||
ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
|
||||
|
||||
/** @brief Create a MixedSesquilinearForm on the given trial and test
|
||||
FiniteElementSpaces, using the same integrators as the
|
||||
MixedBilinearForms @a bfr and @a bfi.
|
||||
|
||||
The FiniteElementSpace pointers are not owned by the newly constructed
|
||||
object.
|
||||
|
||||
The integrators are copied as pointers and they are not owned by the
|
||||
newly constructed MixedSesquilinearForm. */
|
||||
MixedSesquilinearForm(
|
||||
FiniteElementSpace * trial_fes,
|
||||
FiniteElementSpace * test_fes,
|
||||
MixedBilinearForm * bfr,
|
||||
MixedBilinearForm * bfi,
|
||||
ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
|
||||
|
||||
ComplexOperator::Convention GetConvention() const { return conv; }
|
||||
void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
|
||||
|
||||
/// Set the desired assembly level.
|
||||
/** Valid choices are:
|
||||
|
||||
- AssemblyLevel::LEGACY (default)
|
||||
- AssemblyLevel::FULL
|
||||
- AssemblyLevel::PARTIAL
|
||||
- AssemblyLevel::ELEMENT
|
||||
- AssemblyLevel::NONE
|
||||
|
||||
This method must be called before assembly. */
|
||||
void SetAssemblyLevel(AssemblyLevel assembly_level)
|
||||
{
|
||||
mblfr->SetAssemblyLevel(assembly_level);
|
||||
mblfi->SetAssemblyLevel(assembly_level);
|
||||
}
|
||||
|
||||
MixedBilinearForm & real() { return *mblfr; }
|
||||
MixedBilinearForm & imag() { return *mblfi; }
|
||||
const MixedBilinearForm & real() const { return *mblfr; }
|
||||
const MixedBilinearForm & imag() const { return *mblfi; }
|
||||
|
||||
/// Adds new Domain Integrator.
|
||||
void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds new Domain Integrator, restricted to specific attributes.
|
||||
void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> & elem_marker);
|
||||
|
||||
/// Adds new Boundary Integrator.
|
||||
void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/** @brief Adds new boundary Integrator, restricted to specific boundary
|
||||
attributes.
|
||||
|
||||
Assumes ownership of @a bfi.
|
||||
|
||||
The mfem::array @a bdr_marker is stored internally as a pointer to the given
|
||||
mfem::Array<int> object. */
|
||||
void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> & bdr_marker);
|
||||
|
||||
/// Adds new interior Face Integrator. Assumes ownership of @a bfi.
|
||||
void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
|
||||
void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/** @brief Adds new boundary Face Integrator, restricted to specific boundary
|
||||
attributes.
|
||||
|
||||
Assumes ownership of @a bfi.
|
||||
|
||||
The mfem::array @a bdr_marker is stored internally as a pointer to the given
|
||||
mfem::Array<int> object. */
|
||||
void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> & bdr_marker);
|
||||
|
||||
/** @brief Add a trace face integrator. Assumes ownership of @a bfi.
|
||||
|
||||
This type of integrator assembles terms over all faces of the mesh using
|
||||
the face FE from the trial space and the two adjacent volume FEs from
|
||||
the test space. */
|
||||
void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
|
||||
void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
|
||||
void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> &bdr_marker);
|
||||
|
||||
/// Assemble the local matrix
|
||||
void Assemble(int skip_zeros = 1);
|
||||
|
||||
/// Finalizes the matrix initialization.
|
||||
void Finalize(int skip_zeros = 1);
|
||||
|
||||
/// Updates the internal mixed forms with the new finite element space.
|
||||
virtual void Update();
|
||||
|
||||
/** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
|
||||
and imaginary matrices of the form.
|
||||
|
||||
The returned wrapper has to be deleted by the caller, but it does not
|
||||
own the wrapped real and imaginary matrices, which remain owned by
|
||||
this form. */
|
||||
ComplexSparseMatrix *AssembleComplexSparseMatrix();
|
||||
|
||||
/// Return the trial FE space associated with the MixedSesquilinearForm.
|
||||
FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
|
||||
|
||||
/// Read-only access to the associated trial FiniteElementSpace.
|
||||
const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
|
||||
|
||||
/// Return the test FE space associated with the MixedSesquilinearForm.
|
||||
FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
|
||||
|
||||
/// Read-only access to the associated test FiniteElementSpace.
|
||||
const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
|
||||
|
||||
|
||||
void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
|
||||
const Array<int> & ess_test_tdof_list,
|
||||
Vector & x,
|
||||
Vector & b,
|
||||
OperatorHandle & A,
|
||||
Vector & X,
|
||||
Vector & B);
|
||||
|
||||
void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
|
||||
const Array<int> & ess_test_tdof_list,
|
||||
OperatorHandle & A);
|
||||
|
||||
virtual ~MixedSesquilinearForm();
|
||||
};
|
||||
|
||||
#ifdef MFEM_USE_MPI
|
||||
|
||||
/// Class for parallel complex-valued grid function - real + imaginary part
|
||||
@@ -1101,169 +852,6 @@ public:
|
||||
virtual ~ParSesquilinearForm();
|
||||
};
|
||||
|
||||
/** Class for a parallel mixed sesquilinear form
|
||||
|
||||
A mixed sesquilinear form is a generalization of a mixed bilinear form to
|
||||
complex-valued fields. Mixed sesquilinear forms are linear in the second
|
||||
argument but the first argument involves a complex conjugate in the sense
|
||||
that:
|
||||
|
||||
a(alpha u, beta v) = conj(alpha) beta a(u, v)
|
||||
|
||||
The @a convention argument in the class's constructor is documented in the
|
||||
mfem::ComplexOperator class found in linalg/complex_operator.hpp.
|
||||
|
||||
When supplying integrators to the ParMixedSesquilinearForm either the real
|
||||
or imaginary integrator can be NULL. This indicates that the corresponding
|
||||
portion of the complex-valued material coefficient is equal to zero.
|
||||
*/
|
||||
class ParMixedSesquilinearForm
|
||||
{
|
||||
private:
|
||||
ComplexOperator::Convention conv;
|
||||
|
||||
ParMixedBilinearForm * pmblfr;
|
||||
ParMixedBilinearForm * pmblfi;
|
||||
|
||||
/* These methods check if the real/imag parts of the sesqulinear form are
|
||||
not empty */
|
||||
bool RealInteg();
|
||||
bool ImagInteg();
|
||||
|
||||
public:
|
||||
ParMixedSesquilinearForm(
|
||||
ParFiniteElementSpace * trial_fes,
|
||||
ParFiniteElementSpace * test_fes,
|
||||
ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
|
||||
|
||||
/** @brief Create a ParMixedSesquilinearForm on the given trial and test
|
||||
ParFiniteElementSpaces, using the same integrators as the
|
||||
ParMixedBilinearForms @a pbfr and @a pbfi.
|
||||
|
||||
The ParFiniteElementSpace pointers are not owned by the newly
|
||||
constructed object.
|
||||
|
||||
The integrators are copied as pointers and they are not owned by the
|
||||
newly constructed ParMixedSesquilinearForm. */
|
||||
ParMixedSesquilinearForm(
|
||||
ParFiniteElementSpace * trial_fes,
|
||||
ParFiniteElementSpace * test_fes,
|
||||
ParMixedBilinearForm * pbfr,
|
||||
ParMixedBilinearForm * pbfi,
|
||||
ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
|
||||
|
||||
ComplexOperator::Convention GetConvention() const { return conv; }
|
||||
void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
|
||||
|
||||
/// Set the desired assembly level.
|
||||
/** Valid choices are:
|
||||
|
||||
- AssemblyLevel::LEGACY (default)
|
||||
- AssemblyLevel::FULL
|
||||
- AssemblyLevel::PARTIAL
|
||||
- AssemblyLevel::ELEMENT
|
||||
- AssemblyLevel::NONE
|
||||
|
||||
This method must be called before assembly. */
|
||||
void SetAssemblyLevel(AssemblyLevel assembly_level)
|
||||
{
|
||||
pmblfr->SetAssemblyLevel(assembly_level);
|
||||
pmblfi->SetAssemblyLevel(assembly_level);
|
||||
}
|
||||
|
||||
ParMixedBilinearForm & real() { return *pmblfr; }
|
||||
ParMixedBilinearForm & imag() { return *pmblfi; }
|
||||
const ParMixedBilinearForm & real() const { return *pmblfr; }
|
||||
const ParMixedBilinearForm & imag() const { return *pmblfi; }
|
||||
|
||||
/// Adds new Domain Integrator.
|
||||
void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds new Domain Integrator, restricted to specific attributes.
|
||||
void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> & elem_marker);
|
||||
|
||||
/// Adds new Boundary Integrator.
|
||||
void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/** @brief Adds new boundary Integrator, restricted to specific boundary
|
||||
attributes.
|
||||
|
||||
Assumes ownership of @a bfi.
|
||||
|
||||
The mfem::array @a bdr_marker is stored internally as a pointer to the given
|
||||
mfem::Array<int> object. */
|
||||
void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> & bdr_marker);
|
||||
|
||||
/// Adds new interior Face Integrator. Assumes ownership of @a bfi.
|
||||
void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
|
||||
void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/** @brief Adds new boundary Face Integrator, restricted to specific boundary
|
||||
attributes.
|
||||
|
||||
Assumes ownership of @a bfi.
|
||||
|
||||
The mfem::array @a bdr_marker is stored internally as a pointer to the given
|
||||
mfem::Array<int> object. */
|
||||
void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> & bdr_marker);
|
||||
|
||||
/** @brief Add a trace face integrator. Assumes ownership of @a bfi.
|
||||
|
||||
This type of integrator assembles terms over all faces of the mesh using
|
||||
the face FE from the trial space and the two adjacent volume FEs from
|
||||
the test space. */
|
||||
void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
|
||||
void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag);
|
||||
|
||||
/// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
|
||||
void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
|
||||
BilinearFormIntegrator * bfi_imag,
|
||||
Array<int> &bdr_marker);
|
||||
|
||||
/// Assemble the local matrix
|
||||
void Assemble(int skip_zeros = 1);
|
||||
|
||||
/// Finalizes the matrix initialization.
|
||||
void Finalize(int skip_zeros = 1);
|
||||
|
||||
/// Updates the internal mixed forms with the new finite element space.
|
||||
virtual void Update();
|
||||
|
||||
/// Returns the matrix assembled on the true dofs, i.e. P^t A P.
|
||||
/** The returned matrix has to be deleted by the caller. */
|
||||
ComplexHypreParMatrix * ParallelAssemble();
|
||||
|
||||
void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
|
||||
const Array<int> & ess_test_tdof_list,
|
||||
Vector & x,
|
||||
Vector & b,
|
||||
OperatorHandle & A,
|
||||
Vector & X,
|
||||
Vector & B);
|
||||
|
||||
void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
|
||||
const Array<int> & ess_test_tdof_list,
|
||||
OperatorHandle & A);
|
||||
|
||||
virtual ~ParMixedSesquilinearForm();
|
||||
};
|
||||
|
||||
#endif // MFEM_USE_MPI
|
||||
|
||||
}
|
||||
|
||||
@@ -114,10 +114,6 @@ void ConduitDataCollection::Save()
|
||||
n_mesh["fields"][name]);
|
||||
}
|
||||
|
||||
// TODO: in parallel, we need to call ParFiniteElementSpace::ApplyDofSigns
|
||||
// for all ParGridFunction objects before and after saving, see
|
||||
// ParGridFunction::Save.
|
||||
|
||||
// save mesh data
|
||||
SaveMeshAndFields(myid,
|
||||
n_mesh,
|
||||
|
||||
@@ -809,7 +809,7 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
|
||||
|
||||
void ParaViewDataCollectionBase::SetLevelsOfDetail(int levels_of_detail_)
|
||||
{
|
||||
levels_of_detail = std::max(levels_of_detail_, 1);
|
||||
levels_of_detail = levels_of_detail_;
|
||||
}
|
||||
|
||||
void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)
|
||||
@@ -1181,14 +1181,12 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
|
||||
DenseMatrix vval, pmat;
|
||||
std::vector<char> buf;
|
||||
int vec_dim = it->second->VectorDim();
|
||||
int map_type = it->second->FESpace()->GetTypicalFE()->GetMapType();
|
||||
os << "<DataArray type=\"" << GetDataTypeString()
|
||||
<< "\" Name=\"" << it->first
|
||||
<< "\" NumberOfComponents=\"" << vec_dim << "\" "
|
||||
<< VTKComponentLabels(vec_dim) << " "
|
||||
<< "format=\"" << GetDataFormatString() << "\" >" << '\n';
|
||||
if (vec_dim == 1 && (map_type == FiniteElement::VALUE ||
|
||||
map_type == FiniteElement::INTEGRAL))
|
||||
if (vec_dim == 1)
|
||||
{
|
||||
for (int i = 0; i < mesh->GetNE(); i++)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "util.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
/// @brief Assemble element matrix for three dimensional data.
|
||||
///
|
||||
/// Note: In the below layouts, total_trial_op_dim is > 1 if
|
||||
/// there are more than one inputs dependent on the derivative variable.
|
||||
///
|
||||
/// @param A Memory for one element matrix with layout
|
||||
/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
|
||||
/// @param fhat Memory to hold the residual computation with layout
|
||||
/// [test_vdim, test_op_dim, nqp].
|
||||
/// @param qpdc The quadrature point data cache with data layout
|
||||
/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
|
||||
/// @param itod Input Trial Operator Dimension array. If the trial
|
||||
/// operator is not dependent, the dimension is 0 to indicate that.
|
||||
/// @param inputs The input field operator types.
|
||||
/// @param output The output field operator types.
|
||||
/// @param input_dtqmaps The input DofToQuad maps.
|
||||
/// @param output_dtqmap The output DofToQuad maps.
|
||||
/// @param scratch_shmem Scratch shared memory for computations.
|
||||
/// @param q1d The number of quadrature points in one dimension.
|
||||
/// @param td1d The number of trial dofs in one dimension.
|
||||
template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
|
||||
MFEM_HOST_DEVICE void assemble_element_mat_t3d(
|
||||
const DeviceTensor<4, real_t>& A,
|
||||
const DeviceTensor<3, real_t>& fhat,
|
||||
const DeviceTensor<5, const real_t>& qpdc,
|
||||
const DeviceTensor<1, const real_t>& itod,
|
||||
const input_fop_ts& inputs,
|
||||
const output_fop_t& output,
|
||||
const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
|
||||
const DofToQuadMap& output_dtqmap,
|
||||
std::array<DeviceTensor<1>, 6>& scratch_shmem,
|
||||
const int& q1d,
|
||||
const int& td1d)
|
||||
{
|
||||
constexpr int dimension = 3;
|
||||
|
||||
// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
|
||||
const int test_vdim = qpdc.GetShape()[0];
|
||||
const int test_op_dim = qpdc.GetShape()[1];
|
||||
const int trial_vdim = qpdc.GetShape()[2];
|
||||
|
||||
// [num_test_dof, ...]
|
||||
const auto num_test_dof = A.GetShape()[0];
|
||||
|
||||
for (int Jx = 0; Jx < td1d; Jx++)
|
||||
{
|
||||
for (int Jy = 0; Jy < td1d; Jy++)
|
||||
{
|
||||
for (int Jz = 0; Jz < td1d; Jz++)
|
||||
{
|
||||
const int J = Jx + td1d * (Jy + td1d * Jz);
|
||||
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
for (int tv = 0; tv < test_vdim; tv++)
|
||||
{
|
||||
for (int tod = 0; tod < test_op_dim; tod++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
fhat(tv, tod, q) = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MSVC lambda capture workaround
|
||||
[[maybe_unused]] const auto& inputs_ref = inputs;
|
||||
|
||||
int m_offset = 0;
|
||||
for_constexpr<num_inputs>([&](auto s)
|
||||
{
|
||||
using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
|
||||
|
||||
const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
|
||||
if (trial_op_dim == 0)
|
||||
{
|
||||
// This is inside a lambda so we have to return
|
||||
// instead of idiomatic 'continue'.
|
||||
return;
|
||||
}
|
||||
|
||||
auto& B = input_dtqmaps[s].B;
|
||||
auto& G = input_dtqmaps[s].G;
|
||||
|
||||
if constexpr (is_value_fop<fop_t>::value)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
const real_t f = qpdc(i, k, j, m + m_offset, q);
|
||||
fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (is_gradient_fop<fop_t>::value)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
const real_t f = qpdc(i, k, j, m + m_offset, q);
|
||||
if (m == 0)
|
||||
{
|
||||
fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
|
||||
}
|
||||
else if (m == 1)
|
||||
{
|
||||
fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy) * B(qz, 0, Jz);
|
||||
}
|
||||
else if (m == 2)
|
||||
{
|
||||
fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * G(qz, 0, Jz);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
|
||||
MFEM_ABORT("sum factorized sparse matrix assemble routine "
|
||||
"not implemented for field operator");
|
||||
#endif
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
m_offset += trial_op_dim;
|
||||
});
|
||||
|
||||
auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
|
||||
map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
|
||||
scratch_shmem, dimension, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief Assemble element matrix for two dimensional data.
|
||||
///
|
||||
/// Note: In the below layouts, total_trial_op_dim is > 1 if
|
||||
/// there are more than one inputs dependent on the derivative variable.
|
||||
///
|
||||
/// @param A Memory for one element matrix with layout
|
||||
/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
|
||||
/// @param fhat Memory to hold the residual computation with layout
|
||||
/// [test_vdim, test_op_dim, nqp].
|
||||
/// @param qpdc The quadrature point data cache with data layout
|
||||
/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
|
||||
/// @param itod Input Trial Operator Dimension array. If the trial
|
||||
/// operator is not dependent, the dimension is 0 to indicate that.
|
||||
/// @param inputs The input field operator types.
|
||||
/// @param output The output field operator types.
|
||||
/// @param input_dtqmaps The input DofToQuad maps.
|
||||
/// @param output_dtqmap The output DofToQuad maps.
|
||||
/// @param scratch_shmem Scratch shared memory for computations.
|
||||
/// @param q1d The number of quadrature points in one dimension.
|
||||
/// @param td1d The number of trial dofs in one dimension.
|
||||
template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
|
||||
MFEM_HOST_DEVICE void assemble_element_mat_t2d(
|
||||
const DeviceTensor<4, real_t>& A,
|
||||
const DeviceTensor<3, real_t>& fhat,
|
||||
const DeviceTensor<5, const real_t>& qpdc,
|
||||
const DeviceTensor<1, const real_t>& itod,
|
||||
const input_fop_ts& inputs,
|
||||
const output_fop_t& output,
|
||||
const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
|
||||
const DofToQuadMap& output_dtqmap,
|
||||
std::array<DeviceTensor<1>, 6>& scratch_shmem,
|
||||
const int& q1d,
|
||||
const int& td1d)
|
||||
{
|
||||
constexpr int dimension = 2;
|
||||
|
||||
// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
|
||||
const int test_vdim = qpdc.GetShape()[0];
|
||||
const int test_op_dim = qpdc.GetShape()[1];
|
||||
const int trial_vdim = qpdc.GetShape()[2];
|
||||
|
||||
// [num_test_dof, ...]
|
||||
const auto num_test_dof = A.GetShape()[0];
|
||||
|
||||
for (int Jx = 0; Jx < td1d; Jx++)
|
||||
{
|
||||
for (int Jy = 0; Jy < td1d; Jy++)
|
||||
{
|
||||
const int J = Jy + Jx * td1d;
|
||||
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
for (int tv = 0; tv < test_vdim; tv++)
|
||||
{
|
||||
for (int tod = 0; tod < test_op_dim; tod++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
const int q = qy + qx * q1d;
|
||||
fhat(tv, tod, q) = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MSVC lambda capture workaround
|
||||
[[maybe_unused]] const auto& inputs_ref = inputs;
|
||||
|
||||
int m_offset = 0;
|
||||
for_constexpr<num_inputs>([&](auto s)
|
||||
{
|
||||
using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
|
||||
|
||||
const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
|
||||
if (trial_op_dim == 0)
|
||||
{
|
||||
// This is inside a lambda so we have to return
|
||||
// instead of idiomatic 'continue'.
|
||||
return;
|
||||
}
|
||||
|
||||
auto& B = input_dtqmaps[s].B;
|
||||
auto& G = input_dtqmaps[s].G;
|
||||
|
||||
if constexpr (is_value_fop<fop_t>::value)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
const int q = qy + qx * q1d;
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
const real_t f = qpdc(i, k, j, m + m_offset, q);
|
||||
fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (is_gradient_fop<fop_t>::value)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
const int q = qy + qx * q1d;
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
const real_t f = qpdc(i, k, j, m + m_offset, q);
|
||||
if (m == 0)
|
||||
{
|
||||
fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy);
|
||||
}
|
||||
else
|
||||
{
|
||||
fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
|
||||
MFEM_ABORT("sum factorized sparse matrix assemble routine "
|
||||
"not implemented for field operator");
|
||||
#endif
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
m_offset += trial_op_dim;
|
||||
});
|
||||
|
||||
auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
|
||||
map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
|
||||
scratch_shmem, dimension, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief Assemble element matrix for two or three dimensional data.
|
||||
///
|
||||
/// Note: In the below layouts, total_trial_op_dim is > 1 if
|
||||
/// there are more than one inputs dependent on the derivative variable.
|
||||
///
|
||||
/// @param A Memory for one element matrix with layout
|
||||
/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
|
||||
/// @param fhat Memory to hold the residual computation with layout
|
||||
/// [test_vdim, test_op_dim, nqp].
|
||||
/// @param qpdc The quadrature point data cache with data layout
|
||||
/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
|
||||
/// @param itod Input Trial Operator Dimension array. If the trial
|
||||
/// operator is not dependent, the dimension is 0 to indicate that.
|
||||
/// @param inputs The input field operator types.
|
||||
/// @param output The output field operator types.
|
||||
/// @param input_dtqmaps The input DofToQuad maps.
|
||||
/// @param output_dtqmap The output DofToQuad maps.
|
||||
/// @param scratch_shmem Scratch shared memory for computations.
|
||||
/// @param dimension The spatial dimension.
|
||||
/// @param q1d The number of quadrature points in one dimension.
|
||||
/// @param td1d The number of trial dofs in one dimension.
|
||||
/// @param use_sum_factorization Indicator if sum factorization is used.
|
||||
template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
|
||||
MFEM_HOST_DEVICE void assemble_element_mat_naive(
|
||||
const DeviceTensor<4, real_t>& A,
|
||||
const DeviceTensor<3, real_t>& fhat,
|
||||
const DeviceTensor<5, const real_t>& qpdc,
|
||||
const DeviceTensor<1, const real_t>& itod,
|
||||
const input_fop_ts& inputs,
|
||||
const output_fop_t& output,
|
||||
const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
|
||||
const DofToQuadMap& output_dtqmap,
|
||||
std::array<DeviceTensor<1>, 6>& scratch_shmem,
|
||||
const int& dimension,
|
||||
const int& q1d,
|
||||
const int& td1d,
|
||||
const bool& use_sum_factorization)
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 2)
|
||||
{
|
||||
assemble_element_mat_t2d(A, fhat, qpdc, itod, inputs, output,
|
||||
input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
assemble_element_mat_t3d(A, fhat, qpdc, itod, inputs, output,
|
||||
input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
|
||||
MFEM_ABORT("element matrix assemble not implemented for non tensor "
|
||||
"product basis");
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future
|
||||
@@ -1,122 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../util.hpp"
|
||||
#include "../../integrator_ctx.hpp"
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
namespace GlobalQFImpl
|
||||
{
|
||||
|
||||
template<
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t,
|
||||
size_t ninputs = tuple_size<inputs_t>::value,
|
||||
size_t noutputs = tuple_size<outputs_t>::value>
|
||||
struct Action
|
||||
{
|
||||
Action(
|
||||
IntegratorContext ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs) :
|
||||
ctx(ctx),
|
||||
qfunc(qfunc),
|
||||
inputs(inputs),
|
||||
outputs(outputs)
|
||||
{
|
||||
create_fop_to_fd(inputs, ctx.infds, input_to_infd);
|
||||
create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
|
||||
|
||||
check_consistency(inputs, input_to_infd, ctx.infds);
|
||||
check_consistency(outputs, output_to_outfd, ctx.outfds);
|
||||
|
||||
create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
|
||||
create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
|
||||
|
||||
create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
|
||||
create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
|
||||
|
||||
const int nqp = ctx.ir.GetNPoints();
|
||||
gnqp = nqp * ctx.nentities;
|
||||
|
||||
// prepare xq and yq BlockVectors
|
||||
xq_offsets.SetSize(ninputs + 1);
|
||||
xq_offsets[0] = 0;
|
||||
constexpr_for<0, ninputs>([&](auto i)
|
||||
{
|
||||
const auto input = get<i>(inputs);
|
||||
xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
|
||||
});
|
||||
xq_offsets.PartialSum();
|
||||
InitBlockVector(xq, xq_offsets);
|
||||
|
||||
yq_offsets.SetSize(noutputs + 1);
|
||||
yq_offsets[0] = 0;
|
||||
constexpr_for<0, noutputs>([&](auto i)
|
||||
{
|
||||
const auto output = get<i>(outputs);
|
||||
yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
|
||||
});
|
||||
yq_offsets.PartialSum();
|
||||
InitBlockVector(yq, yq_offsets);
|
||||
}
|
||||
|
||||
void operator()(
|
||||
const std::vector<Vector *> &xe,
|
||||
std::vector<Vector *> &ye) const
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
// E -> Q
|
||||
interpolate(input_to_infd, input_bases, xe, xq);
|
||||
|
||||
// Q -> Q
|
||||
static_assert(
|
||||
detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
|
||||
"qfunc signature not supported by default backend Action");
|
||||
|
||||
detail::call_qfunc(
|
||||
qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
|
||||
std::make_index_sequence<ninputs> {},
|
||||
std::make_index_sequence<noutputs> {});
|
||||
|
||||
// Q -> E
|
||||
integrate(output_to_outfd, output_bases, yq, ye);
|
||||
}
|
||||
|
||||
IntegratorContext ctx;
|
||||
qfunc_t qfunc;
|
||||
inputs_t inputs;
|
||||
outputs_t outputs;
|
||||
|
||||
std::array<size_t, ninputs> input_to_infd;
|
||||
std::array<size_t, noutputs> output_to_outfd;
|
||||
|
||||
std::array<FieldBasis, ninputs> input_bases;
|
||||
std::array<FieldBasis, noutputs> output_bases;
|
||||
|
||||
std::array<std::vector<int>, ninputs> input_qlayouts;
|
||||
std::array<std::vector<int>, noutputs> output_qlayouts;
|
||||
|
||||
int gnqp = 0;
|
||||
Array<int> xq_offsets, yq_offsets;
|
||||
mutable BlockVector xq, yq;
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
#include "../util.hpp"
|
||||
#include <utility>
|
||||
|
||||
namespace mfem::future::GlobalQFImpl
|
||||
{
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t,
|
||||
size_t ninputs = tuple_size<inputs_t>::value,
|
||||
size_t noutputs = tuple_size<outputs_t>::value>
|
||||
struct DerivativeAction
|
||||
{
|
||||
using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
|
||||
|
||||
DerivativeAction(
|
||||
IntegratorContext ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs):
|
||||
ctx(ctx), qfunc(qfunc),
|
||||
qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
|
||||
inputs(inputs), outputs(outputs)
|
||||
{
|
||||
create_fop_to_fd(inputs, ctx.infds, input_to_infd);
|
||||
create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
|
||||
|
||||
check_consistency(inputs, input_to_infd, ctx.infds);
|
||||
check_consistency(outputs, output_to_outfd, ctx.outfds);
|
||||
|
||||
create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
|
||||
create_fieldbases(
|
||||
outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
|
||||
|
||||
create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
|
||||
create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
|
||||
|
||||
const int nqp = ctx.ir.GetNPoints();
|
||||
gnqp = nqp * ctx.nentities;
|
||||
|
||||
xq_offsets.SetSize(ninputs + 1);
|
||||
xq_offsets[0] = 0;
|
||||
constexpr_for<0, ninputs>([&](auto i)
|
||||
{
|
||||
const auto input = get<i>(inputs);
|
||||
xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
|
||||
});
|
||||
xq_offsets.PartialSum();
|
||||
InitBlockVector(xq, xq_offsets);
|
||||
|
||||
yq_offsets.SetSize(noutputs + 1);
|
||||
yq_offsets[0] = 0;
|
||||
constexpr_for<0, noutputs>([&](auto i)
|
||||
{
|
||||
const auto output = get<i>(outputs);
|
||||
yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
|
||||
});
|
||||
yq_offsets.PartialSum();
|
||||
InitBlockVector(yq, yq_offsets);
|
||||
|
||||
// Shadow blocks use the same offsets as xq so tensor_array views
|
||||
shadow_xq_offsets.SetSize(xq_offsets.Size());
|
||||
shadow_xq_offsets = xq_offsets;
|
||||
InitBlockVector(shadow_xq, shadow_xq_offsets);
|
||||
|
||||
dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
|
||||
|
||||
const size_t direction_fd_idx = FindIdx(derivative_id, ctx.infds);
|
||||
MFEM_ASSERT(direction_fd_idx != SIZE_MAX,
|
||||
"derivative direction field not found in infds");
|
||||
direction_fd = ctx.infds[direction_fd_idx];
|
||||
}
|
||||
|
||||
void operator()(
|
||||
const std::vector<Vector *> &xe,
|
||||
const Vector *de,
|
||||
std::vector<Vector *> &ye)
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
// E -> Q
|
||||
interpolate(input_to_infd, input_bases, xe, xq);
|
||||
|
||||
constexpr auto input_active =
|
||||
detail::make_activity_map<derivative_id>(inputs_t{});
|
||||
|
||||
MFEM_ASSERT(de != nullptr, "derivative direction vector is null");
|
||||
restriction(direction_fd, direction_rcache, *de, direction_e,
|
||||
dof_ordering);
|
||||
|
||||
shadow_xq = 0.0;
|
||||
shadow_xq.SyncToBlocks();
|
||||
constexpr_for<0, ninputs>([&](auto i)
|
||||
{
|
||||
if (!input_active[i]) { return; }
|
||||
input_bases[i].forward(direction_e, shadow_xq.GetBlock(i));
|
||||
});
|
||||
|
||||
static_assert(detail::supports_tensor_array_qfunc<qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::value,
|
||||
"qfunc signature not supported by default backend Action");
|
||||
|
||||
// Q -> Q
|
||||
yq = 0.0;
|
||||
yq.SyncToBlocks();
|
||||
if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
|
||||
{
|
||||
detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t, inputs_t,
|
||||
outputs_t>(
|
||||
qfunc,
|
||||
qfunc_shadow,
|
||||
xq,
|
||||
shadow_xq,
|
||||
yq,
|
||||
gnqp,
|
||||
input_qlayouts,
|
||||
output_qlayouts,
|
||||
std::make_index_sequence<ninputs> {},
|
||||
std::make_index_sequence<noutputs> {});
|
||||
}
|
||||
else
|
||||
{
|
||||
detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
|
||||
qfunc,
|
||||
xq,
|
||||
shadow_xq,
|
||||
yq,
|
||||
gnqp,
|
||||
input_qlayouts,
|
||||
output_qlayouts,
|
||||
std::make_index_sequence<ninputs> {},
|
||||
std::make_index_sequence<noutputs> {});
|
||||
}
|
||||
|
||||
// Q -> E
|
||||
integrate(output_to_outfd, output_bases, yq, ye);
|
||||
}
|
||||
|
||||
IntegratorContext ctx;
|
||||
qfunc_t qfunc;
|
||||
qfunc_shadow_t qfunc_shadow;
|
||||
inputs_t inputs;
|
||||
outputs_t outputs;
|
||||
|
||||
std::array<size_t, ninputs> input_to_infd;
|
||||
std::array<size_t, noutputs> output_to_outfd;
|
||||
|
||||
std::array<FieldBasis, ninputs> input_bases;
|
||||
std::array<FieldBasis, noutputs> output_bases;
|
||||
|
||||
std::array<std::vector<int>, ninputs> input_qlayouts;
|
||||
std::array<std::vector<int>, noutputs> output_qlayouts;
|
||||
|
||||
int gnqp = 0;
|
||||
Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
|
||||
mutable BlockVector xq, shadow_xq, yq;
|
||||
|
||||
FieldDescriptor direction_fd;
|
||||
ElementDofOrdering dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
|
||||
mutable Vector direction_e;
|
||||
mutable RestrictionCache<Entity::Element> direction_rcache;
|
||||
};
|
||||
|
||||
} // namespace mfem::future::GlobalQFImpl
|
||||
@@ -1,242 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
#include "../util.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace mfem::future::GlobalQFImpl
|
||||
{
|
||||
|
||||
// Q-function-shape-agnostic cached forward apply (J·v)
|
||||
template<
|
||||
int derivative_id,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
struct DerivativeApply
|
||||
{
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
|
||||
DerivativeApply(
|
||||
IntegratorContext ctx,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache):
|
||||
ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
|
||||
qp_cache(qp_cache)
|
||||
{
|
||||
create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
|
||||
create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
|
||||
|
||||
check_consistency(this->inputs, input_to_infd, ctx.infds);
|
||||
check_consistency(this->outputs, output_to_outfd, ctx.outfds);
|
||||
|
||||
create_fieldbases(
|
||||
this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
|
||||
create_fieldbases(
|
||||
this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
|
||||
|
||||
const int nqp = ctx.ir.GetNPoints();
|
||||
const int ne = ctx.nentities;
|
||||
num_qp = nqp;
|
||||
gnqp = nqp * ne;
|
||||
|
||||
// Precompute Q-space BlockVector layouts
|
||||
dir_q_offsets.SetSize(n_inputs + 1);
|
||||
dir_q_offsets[0] = 0;
|
||||
constexpr_for<0, n_inputs>([&](auto i)
|
||||
{
|
||||
dir_q_offsets[i + 1] =
|
||||
dir_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
|
||||
});
|
||||
InitBlockVector(dir_q_local, dir_q_offsets);
|
||||
|
||||
result_q_offsets.SetSize(n_outputs + 1);
|
||||
result_q_offsets[0] = 0;
|
||||
constexpr_for<0, n_outputs>([&](auto i)
|
||||
{
|
||||
result_q_offsets[i + 1] =
|
||||
result_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
|
||||
});
|
||||
InitBlockVector(result_q_local, result_q_offsets);
|
||||
|
||||
// Cache layout metadata (must match DerivativeSetup)
|
||||
residual_size_on_qp = 0;
|
||||
trial_vdim = 0;
|
||||
total_trial_op_dim = 0;
|
||||
|
||||
constexpr auto activity =
|
||||
detail::make_activity_map<derivative_id>(inputs_t{});
|
||||
|
||||
constexpr_for<0, n_inputs>([&](auto i)
|
||||
{
|
||||
if (!activity[i]) { return; }
|
||||
const auto &fop = get<i>(this->inputs);
|
||||
trial_vdim = fop.vdim;
|
||||
total_trial_op_dim += fop.size_on_qp / fop.vdim;
|
||||
});
|
||||
|
||||
constexpr_for<0, n_outputs>([&](auto i)
|
||||
{ residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
|
||||
residual_size_on_qp *= trial_vdim * total_trial_op_dim;
|
||||
}
|
||||
|
||||
void operator()(
|
||||
const std::vector<Vector *> & /*xe*/,
|
||||
const Vector *direction_l,
|
||||
std::vector<Vector *> &ye) const
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
MFEM_ASSERT(direction_l != nullptr,
|
||||
"Global DerivativeApply: direction vector is null");
|
||||
|
||||
// Re-zero pre-allocated Q temporaries
|
||||
dir_q_local = 0.0;
|
||||
dir_q_local.SyncToBlocks();
|
||||
result_q_local = 0.0;
|
||||
result_q_local.SyncToBlocks();
|
||||
|
||||
// Restrict trial direction from the derivative field
|
||||
size_t in_fd = SIZE_MAX;
|
||||
constexpr_for<0, n_inputs>([&](auto i)
|
||||
{
|
||||
if (get<i>(inputs).GetFieldId() == derivative_id)
|
||||
{
|
||||
in_fd = input_to_infd[i.value];
|
||||
}
|
||||
});
|
||||
MFEM_ASSERT(in_fd != SIZE_MAX,
|
||||
"DerivativeApply: derivative field not found among inputs");
|
||||
|
||||
const auto &fd = ctx.infds[in_fd];
|
||||
|
||||
Vector dir_e;
|
||||
restriction(fd, direction_rcache, *direction_l, dir_e,
|
||||
ElementDofOrdering::LEXICOGRAPHIC);
|
||||
|
||||
// Forward the trial direction into active input Q block
|
||||
constexpr_for<0, n_inputs>([&](auto s)
|
||||
{
|
||||
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
|
||||
input_bases[s.value].forward(dir_e, dir_q_local.GetBlock(s.value));
|
||||
});
|
||||
|
||||
const real_t *cache_ptr = qp_cache.Read();
|
||||
const int res_sz = residual_size_on_qp;
|
||||
const int gnqp_local = gnqp;
|
||||
const int num_qp_local = num_qp;
|
||||
const int trial_vdim_local = trial_vdim;
|
||||
const int total_trial_op_dim_local = total_trial_op_dim;
|
||||
|
||||
constexpr_for<0, n_outputs>([&](auto o)
|
||||
{
|
||||
const int tv_o = get<o>(outputs).vdim;
|
||||
const int to_o = get<o>(outputs).size_on_qp / tv_o;
|
||||
const int out_base = [&]
|
||||
{
|
||||
int off = 0;
|
||||
constexpr_for<0, o.value>([&](auto prev)
|
||||
{ off += get<prev>(outputs).size_on_qp; });
|
||||
return off;
|
||||
}();
|
||||
|
||||
real_t *res_o = result_q_local.GetBlock(o.value).ReadWrite();
|
||||
|
||||
int m_offset = 0;
|
||||
constexpr_for<0, n_inputs>([&](auto s)
|
||||
{
|
||||
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
|
||||
|
||||
const int tv = get<s>(inputs).vdim;
|
||||
const int to = get<s>(inputs).size_on_qp / tv;
|
||||
const real_t *dir_s = dir_q_local.GetBlock(s.value).Read();
|
||||
|
||||
mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
|
||||
{
|
||||
// Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
|
||||
// read adjacent addresses for a fixed cache_idx.
|
||||
const int cache_base =
|
||||
(gq % num_qp_local) +
|
||||
num_qp_local * res_sz * (gq / num_qp_local);
|
||||
|
||||
for (int j = 0; j < tv; ++j)
|
||||
{
|
||||
for (int m = 0; m < to; ++m)
|
||||
{
|
||||
const real_t v = dir_s[(j * to + m) + (tv * to) * gq];
|
||||
const int m_global = m + m_offset;
|
||||
|
||||
for (int i = 0; i < tv_o; ++i)
|
||||
{
|
||||
for (int k = 0; k < to_o; ++k)
|
||||
{
|
||||
const int out_comp = out_base + i * to_o + k;
|
||||
|
||||
const int cache_idx =
|
||||
out_comp * trial_vdim_local * total_trial_op_dim_local +
|
||||
j * total_trial_op_dim_local + m_global;
|
||||
|
||||
const real_t c =
|
||||
cache_ptr[cache_base + num_qp_local * cache_idx];
|
||||
res_o[(i * to_o + k) + (tv_o * to_o) * gq] += c * v;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
m_offset += to;
|
||||
});
|
||||
});
|
||||
|
||||
result_q_local.SyncToBlocks();
|
||||
|
||||
// Map result Q back to output fields
|
||||
constexpr_for<0, n_outputs>([&](auto o)
|
||||
{
|
||||
const size_t out_fd = output_to_outfd[o.value];
|
||||
output_bases[o.value].transpose(result_q_local.GetBlock(o.value),
|
||||
*ye[out_fd]);
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
IntegratorContext ctx;
|
||||
inputs_t inputs;
|
||||
outputs_t outputs;
|
||||
const Vector &qp_cache;
|
||||
|
||||
std::array<size_t, n_inputs> input_to_infd;
|
||||
std::array<size_t, n_outputs> output_to_outfd;
|
||||
|
||||
std::array<FieldBasis, n_inputs> input_bases;
|
||||
std::array<FieldBasis, n_outputs> output_bases;
|
||||
|
||||
int gnqp = 0;
|
||||
int num_qp = 0;
|
||||
|
||||
Array<int> dir_q_offsets;
|
||||
Array<int> result_q_offsets;
|
||||
mutable BlockVector dir_q_local;
|
||||
mutable BlockVector result_q_local;
|
||||
mutable RestrictionCache<Entity::Element> direction_rcache;
|
||||
|
||||
int residual_size_on_qp = 0;
|
||||
int trial_vdim = 0;
|
||||
int total_trial_op_dim = 0;
|
||||
};
|
||||
|
||||
} // namespace mfem::future::GlobalQFImpl
|
||||
@@ -1,260 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
#include "../util.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace mfem::future::GlobalQFImpl
|
||||
{
|
||||
|
||||
// Q-function-shape-agnostic cached transpose apply (Jᵀ·w)
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
struct DerivativeApplyTranspose
|
||||
{
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
|
||||
DerivativeApplyTranspose(
|
||||
IntegratorContext ctx,
|
||||
qfunc_t /*qfunc*/,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache):
|
||||
ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
|
||||
qp_cache(qp_cache), dir_out_l(n_outputs), dir_out_e(n_outputs)
|
||||
{
|
||||
create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
|
||||
create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
|
||||
|
||||
check_consistency(this->inputs, input_to_infd, ctx.infds);
|
||||
check_consistency(this->outputs, output_to_outfd, ctx.outfds);
|
||||
|
||||
create_fieldbases(
|
||||
this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
|
||||
create_fieldbases(
|
||||
this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
|
||||
|
||||
const int nqp = ctx.ir.GetNPoints();
|
||||
const int ne = ctx.nentities;
|
||||
num_qp = nqp;
|
||||
gnqp = nqp * ne;
|
||||
|
||||
// Precompute Q-space BlockVector layouts
|
||||
dir_q_offsets.SetSize(n_outputs + 1);
|
||||
dir_q_offsets[0] = 0;
|
||||
constexpr_for<0, n_outputs>([&](auto i)
|
||||
{
|
||||
dir_q_offsets[i + 1] =
|
||||
dir_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
|
||||
});
|
||||
InitBlockVector(dir_q_local, dir_q_offsets);
|
||||
|
||||
result_q_offsets.SetSize(n_inputs + 1);
|
||||
result_q_offsets[0] = 0;
|
||||
constexpr_for<0, n_inputs>([&](auto i)
|
||||
{
|
||||
result_q_offsets[i + 1] =
|
||||
result_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
|
||||
});
|
||||
InitBlockVector(result_q_local, result_q_offsets);
|
||||
|
||||
// Cache layout metadata
|
||||
residual_size_on_qp = 0;
|
||||
trial_vdim = 0;
|
||||
total_trial_op_dim = 0;
|
||||
|
||||
constexpr auto activity =
|
||||
detail::make_activity_map<derivative_id>(inputs_t{});
|
||||
|
||||
constexpr_for<0, n_inputs>([&](auto i)
|
||||
{
|
||||
if (!activity[i]) { return; }
|
||||
const auto &fop = get<i>(this->inputs);
|
||||
trial_vdim = fop.vdim;
|
||||
total_trial_op_dim += fop.size_on_qp / fop.vdim;
|
||||
});
|
||||
|
||||
constexpr_for<0, n_outputs>([&](auto i)
|
||||
{ residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
|
||||
residual_size_on_qp *= trial_vdim * total_trial_op_dim;
|
||||
}
|
||||
|
||||
void operator()(
|
||||
const std::vector<Vector *> & /*xe*/,
|
||||
const Vector *direction_l,
|
||||
std::vector<Vector *> &ye) const
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
MFEM_ASSERT(direction_l != nullptr,
|
||||
"Global DerivativeApplyTranspose: direction vector is null");
|
||||
|
||||
// Re-zero the pre-allocated Q temporaries
|
||||
dir_q_local = 0.0;
|
||||
result_q_local = 0.0;
|
||||
dir_q_local.SyncToBlocks();
|
||||
result_q_local.SyncToBlocks();
|
||||
|
||||
// Bring test cotangent to quadrature points
|
||||
pull_output_cotangents_to_q(direction_l, dir_q_local);
|
||||
|
||||
// Contract qp_cache with test directions at quadrature points
|
||||
const real_t *cache_ptr = qp_cache.Read();
|
||||
const int res_sz = residual_size_on_qp;
|
||||
const int gnqp_local = gnqp;
|
||||
const int num_qp_local = num_qp;
|
||||
const int trial_vdim_local = trial_vdim;
|
||||
const int total_trial_op_dim_local = total_trial_op_dim;
|
||||
|
||||
constexpr_for<0, n_outputs>([&](auto o)
|
||||
{
|
||||
const int tv_o = get<o>(outputs).vdim;
|
||||
const int to_o = get<o>(outputs).size_on_qp / tv_o;
|
||||
const int out_base = [&]
|
||||
{
|
||||
int off = 0;
|
||||
constexpr_for<0, o.value>([&](auto prev)
|
||||
{ off += get<prev>(outputs).size_on_qp; });
|
||||
return off;
|
||||
}();
|
||||
|
||||
const int size_o = get<o>(outputs).size_on_qp;
|
||||
const real_t *dir_o = dir_q_local.GetBlock(o.value).Read();
|
||||
|
||||
int m_offset = 0;
|
||||
constexpr_for<0, n_inputs>([&](auto s)
|
||||
{
|
||||
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
|
||||
|
||||
const int size_s = get<s>(inputs).size_on_qp;
|
||||
const int to_s = size_s / trial_vdim_local;
|
||||
real_t *res_s = result_q_local.GetBlock(s.value).ReadWrite();
|
||||
|
||||
mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
|
||||
{
|
||||
// Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
|
||||
// read adjacent addresses for a fixed cache_idx.
|
||||
const int cache_base =
|
||||
(gq % num_qp_local) +
|
||||
num_qp_local * res_sz * (gq / num_qp_local);
|
||||
|
||||
for (int i = 0; i < tv_o; ++i)
|
||||
{
|
||||
for (int k = 0; k < to_o; ++k)
|
||||
{
|
||||
const int out_comp = out_base + i * to_o + k;
|
||||
const real_t w = dir_o[(i * to_o + k) + size_o * gq];
|
||||
|
||||
for (int j = 0; j < trial_vdim_local; ++j)
|
||||
{
|
||||
for (int m = 0; m < to_s; ++m)
|
||||
{
|
||||
const int m_global = m + m_offset;
|
||||
const int cache_idx =
|
||||
out_comp * trial_vdim_local * total_trial_op_dim_local +
|
||||
j * total_trial_op_dim_local + m_global;
|
||||
|
||||
const real_t c =
|
||||
cache_ptr[cache_base + num_qp_local * cache_idx];
|
||||
res_s[(j * to_s + m) + size_s * gq] += c * w;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
m_offset += to_s;
|
||||
});
|
||||
});
|
||||
|
||||
// Map result Q back to the trial (input) fields
|
||||
constexpr_for<0, n_inputs>([&](auto s)
|
||||
{
|
||||
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
|
||||
|
||||
const size_t in_fd = input_to_infd[s.value];
|
||||
input_bases[s.value].transpose(
|
||||
result_q_local.GetBlock(s.value), *ye[in_fd]);
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
IntegratorContext ctx;
|
||||
inputs_t inputs;
|
||||
outputs_t outputs;
|
||||
const Vector &qp_cache;
|
||||
|
||||
std::array<size_t, n_inputs> input_to_infd;
|
||||
std::array<size_t, n_outputs> output_to_outfd;
|
||||
|
||||
std::array<FieldBasis, n_inputs> input_bases;
|
||||
std::array<FieldBasis, n_outputs> output_bases;
|
||||
|
||||
int gnqp = 0;
|
||||
int num_qp = 0;
|
||||
|
||||
// Pre-allocated Q-space temporaries
|
||||
Array<int> dir_q_offsets;
|
||||
Array<int> result_q_offsets;
|
||||
mutable BlockVector dir_q_local;
|
||||
mutable BlockVector result_q_local;
|
||||
|
||||
// Pre-allocated owning storage for output cotangent temporaries
|
||||
mutable std::array<Vector, n_outputs> dir_out_l_owned;
|
||||
mutable std::array<Vector, n_outputs> dir_out_e_owned;
|
||||
mutable std::vector<Vector *> dir_out_l;
|
||||
mutable std::vector<Vector *> dir_out_e;
|
||||
mutable RestrictionCache<Entity::Element> out_rcache;
|
||||
|
||||
int residual_size_on_qp = 0;
|
||||
int trial_vdim = 0;
|
||||
int total_trial_op_dim = 0;
|
||||
|
||||
/// Pull output cotangents from L-space into the pre-allocated Q BlockVector
|
||||
void pull_output_cotangents_to_q(const Vector *direction_l,
|
||||
BlockVector &dir_q) const
|
||||
{
|
||||
int l_offset = 0;
|
||||
constexpr_for<0, n_outputs>([&](auto i)
|
||||
{
|
||||
const size_t outfd = output_to_outfd[i];
|
||||
const auto &fd = ctx.outfds[outfd];
|
||||
const int l_size = GetVSize(fd);
|
||||
|
||||
dir_out_l_owned[i] =
|
||||
Vector(*const_cast<Vector *>(direction_l), l_offset, l_size);
|
||||
dir_out_e_owned[i].SetSize(0);
|
||||
dir_out_e_owned[i].UseDevice(true);
|
||||
|
||||
dir_out_l[i] = &dir_out_l_owned[i];
|
||||
dir_out_e[i] = &dir_out_e_owned[i];
|
||||
l_offset += l_size;
|
||||
});
|
||||
|
||||
restriction(ctx.outfds, out_rcache, dir_out_l, dir_out_e);
|
||||
|
||||
constexpr_for<0, n_outputs>([&](auto i)
|
||||
{
|
||||
output_bases[i.value].forward(*dir_out_e[i], dir_q.GetBlock(i.value));
|
||||
});
|
||||
dir_q.SyncToBlocks();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mfem::future::GlobalQFImpl
|
||||
@@ -1,244 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
#include "../util.hpp"
|
||||
#include <utility>
|
||||
|
||||
namespace mfem::future::GlobalQFImpl
|
||||
{
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t,
|
||||
size_t ninputs = tuple_size<inputs_t>::value,
|
||||
size_t noutputs = tuple_size<outputs_t>::value>
|
||||
struct DerivativeSetup
|
||||
{
|
||||
using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
|
||||
|
||||
DerivativeSetup(
|
||||
IntegratorContext ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
Vector &qp_cache) :
|
||||
ctx(ctx),
|
||||
qfunc(qfunc),
|
||||
qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
|
||||
inputs(inputs),
|
||||
outputs(outputs),
|
||||
qp_cache(qp_cache)
|
||||
{
|
||||
create_fop_to_fd(inputs, ctx.infds, input_to_infd);
|
||||
create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
|
||||
|
||||
check_consistency(inputs, input_to_infd, ctx.infds);
|
||||
check_consistency(outputs, output_to_outfd, ctx.outfds);
|
||||
|
||||
create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
|
||||
create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
|
||||
|
||||
create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
|
||||
create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
|
||||
|
||||
const int nqp = ctx.ir.GetNPoints();
|
||||
num_qp = nqp;
|
||||
nentities = ctx.nentities;
|
||||
gnqp = nqp * nentities;
|
||||
|
||||
xq_offsets.SetSize(ninputs + 1);
|
||||
xq_offsets[0] = 0;
|
||||
constexpr_for<0, ninputs>([&](auto i)
|
||||
{
|
||||
xq_offsets[i + 1] = nqp * get<i>(inputs).size_on_qp * nentities;
|
||||
});
|
||||
xq_offsets.PartialSum();
|
||||
InitBlockVector(xq, xq_offsets);
|
||||
|
||||
shadow_xq_offsets.SetSize(xq_offsets.Size());
|
||||
shadow_xq_offsets = xq_offsets;
|
||||
InitBlockVector(shadow_xq, shadow_xq_offsets);
|
||||
|
||||
yq_offsets.SetSize(noutputs + 1);
|
||||
yq_offsets[0] = 0;
|
||||
constexpr_for<0, noutputs>([&](auto o)
|
||||
{
|
||||
yq_offsets[o + 1] = nqp * get<o>(outputs).size_on_qp * nentities;
|
||||
});
|
||||
yq_offsets.PartialSum();
|
||||
InitBlockVector(yq, yq_offsets);
|
||||
|
||||
total_out_size_on_qp = 0;
|
||||
constexpr_for<0, noutputs>([&](auto o)
|
||||
{
|
||||
total_out_size_on_qp += get<o>(outputs).size_on_qp;
|
||||
out_vdim[o] = get<o>(outputs).vdim;
|
||||
out_op_dim[o] = get<o>(outputs).size_on_qp / get<o>(outputs).vdim;
|
||||
});
|
||||
|
||||
activity_map = detail::make_activity_map<derivative_id>(inputs_t {});
|
||||
|
||||
trial_vdim = 0;
|
||||
total_trial_op_dim = 0;
|
||||
constexpr_for<0, ninputs>([&](auto i)
|
||||
{
|
||||
if (!activity_map[i]) { return; }
|
||||
const auto inp = get<i>(inputs);
|
||||
trial_vdim = inp.vdim;
|
||||
total_trial_op_dim += inp.size_on_qp / inp.vdim;
|
||||
});
|
||||
|
||||
constexpr_for<0, ninputs>([&](auto i)
|
||||
{
|
||||
input_size_on_qp_arr[i] = get<i>(inputs).size_on_qp;
|
||||
});
|
||||
|
||||
residual_size_on_qp = total_out_size_on_qp * trial_vdim * total_trial_op_dim;
|
||||
qp_cache.SetSize(residual_size_on_qp * num_qp * nentities);
|
||||
qp_cache.UseDevice(true);
|
||||
}
|
||||
|
||||
void operator()(const std::vector<Vector *> &xe)
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
interpolate(input_to_infd, input_bases, xe, xq);
|
||||
|
||||
const int gnqp_local = gnqp;
|
||||
const int num_qp_local = num_qp;
|
||||
const int trial_vdim_local = trial_vdim;
|
||||
const int total_trial_op_dim_local = total_trial_op_dim;
|
||||
const int residual_size_local = residual_size_on_qp;
|
||||
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
int m_offset = 0;
|
||||
constexpr_for<0, ninputs>([&](auto s)
|
||||
{
|
||||
if (!activity_map[s]) { return; }
|
||||
|
||||
const int input_vdim_s = get<s>(inputs).vdim;
|
||||
const int input_size_s = input_size_on_qp_arr[s];
|
||||
const int trial_op_dim_s = input_size_s / input_vdim_s;
|
||||
|
||||
for (int m = 0; m < trial_op_dim_s; m++)
|
||||
{
|
||||
shadow_xq = 0.0;
|
||||
shadow_xq.SyncToBlocks();
|
||||
|
||||
// Set component (j + input_vdim_s * m) to 1 at all QPs
|
||||
const int c_shadow = j + input_vdim_s * m;
|
||||
real_t *shadow_ptr = shadow_xq.GetBlock(s.value).ReadWrite();
|
||||
mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
|
||||
{
|
||||
shadow_ptr[c_shadow + input_size_s * gq] = 1.0;
|
||||
});
|
||||
|
||||
yq = 0.0;
|
||||
yq.SyncToBlocks();
|
||||
|
||||
if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
|
||||
{
|
||||
detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t,
|
||||
inputs_t, outputs_t>(
|
||||
qfunc, qfunc_shadow, xq, shadow_xq, yq, gnqp,
|
||||
input_qlayouts, output_qlayouts,
|
||||
std::make_index_sequence<ninputs> {},
|
||||
std::make_index_sequence<noutputs> {});
|
||||
}
|
||||
else
|
||||
{
|
||||
detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
|
||||
qfunc, xq, shadow_xq, yq, gnqp,
|
||||
input_qlayouts, output_qlayouts,
|
||||
std::make_index_sequence<ninputs> {},
|
||||
std::make_index_sequence<noutputs> {});
|
||||
}
|
||||
|
||||
real_t *cache_d = qp_cache.ReadWrite();
|
||||
|
||||
// Write yq into the cache column
|
||||
const int m_global = m + m_offset;
|
||||
const int j_cur = j;
|
||||
int out_offset = 0;
|
||||
constexpr_for<0, noutputs>([&](auto o)
|
||||
{
|
||||
const int test_vdim_o = out_vdim[o];
|
||||
const int test_op_dim_o = out_op_dim[o];
|
||||
const int yq_out_size = test_vdim_o * test_op_dim_o;
|
||||
const int out_offset_o = out_offset;
|
||||
const real_t *yq_d = yq.GetBlock(o.value).Read();
|
||||
|
||||
// The cache is (q, cache_idx, e) with the quadrature index
|
||||
// fastest, so gq is the fastest-varying thread index to keep
|
||||
// the stores coalesced.
|
||||
mfem::forall(gnqp_local * yq_out_size, [=] MFEM_HOST_DEVICE(int idx)
|
||||
{
|
||||
const int gq = idx % gnqp_local;
|
||||
const int c_out = idx / gnqp_local;
|
||||
const int q = gq % num_qp_local;
|
||||
const int entity = gq / num_qp_local;
|
||||
const int out_comp = out_offset_o + c_out;
|
||||
const int cache_idx =
|
||||
out_comp * trial_vdim_local * total_trial_op_dim_local +
|
||||
j_cur * total_trial_op_dim_local +
|
||||
m_global;
|
||||
cache_d[q + num_qp_local *
|
||||
(cache_idx + residual_size_local * entity)] =
|
||||
yq_d[c_out + yq_out_size * gq];
|
||||
});
|
||||
out_offset += yq_out_size;
|
||||
});
|
||||
}
|
||||
m_offset += trial_op_dim_s;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
IntegratorContext ctx;
|
||||
qfunc_t qfunc;
|
||||
qfunc_shadow_t qfunc_shadow;
|
||||
inputs_t inputs;
|
||||
outputs_t outputs;
|
||||
Vector &qp_cache;
|
||||
|
||||
std::array<size_t, ninputs> input_to_infd;
|
||||
std::array<size_t, noutputs> output_to_outfd;
|
||||
|
||||
std::array<FieldBasis, ninputs> input_bases;
|
||||
std::array<FieldBasis, noutputs> output_bases;
|
||||
|
||||
std::array<std::vector<int>, ninputs> input_qlayouts;
|
||||
std::array<std::vector<int>, noutputs> output_qlayouts;
|
||||
|
||||
int gnqp = 0;
|
||||
int num_qp = 0;
|
||||
int nentities = 0;
|
||||
|
||||
Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
|
||||
mutable BlockVector xq, shadow_xq, yq;
|
||||
|
||||
int total_out_size_on_qp = 0;
|
||||
int trial_vdim = 0;
|
||||
int total_trial_op_dim = 0;
|
||||
int residual_size_on_qp = 0;
|
||||
|
||||
std::array<int, noutputs> out_vdim {};
|
||||
std::array<int, noutputs> out_op_dim {};
|
||||
std::array<int, ninputs> input_size_on_qp_arr {};
|
||||
std::array<bool, ninputs> activity_map {};
|
||||
};
|
||||
|
||||
} // namespace mfem::future::GlobalQFImpl
|
||||
@@ -1,219 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "action.hpp"
|
||||
#include "derivative_action.hpp"
|
||||
#include "derivative_setup.hpp"
|
||||
|
||||
#include "derivative_apply_transpose.hpp"
|
||||
|
||||
#include "../local_qf/derivative_apply.hpp"
|
||||
#include "../local_qf/derivative_assemble.hpp"
|
||||
#include "../local_qf/derivative_assemble_diagonal.hpp"
|
||||
|
||||
#include "../scratch_bank.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
struct LocalQFShapeArg
|
||||
{
|
||||
using type = std::remove_const_t<T>&;
|
||||
};
|
||||
|
||||
template <typename scalar_t, int ndims, int... tensor_sizes>
|
||||
struct LocalQFShapeArg<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
|
||||
{
|
||||
using scalar_type = std::remove_const_t<scalar_t>;
|
||||
using type = std::conditional_t<
|
||||
sizeof...(tensor_sizes) == 0,
|
||||
scalar_type,
|
||||
tensor<scalar_type, tensor_sizes...>>&;
|
||||
};
|
||||
|
||||
template <typename scalar_t, int... tensor_sizes>
|
||||
struct LocalQFShapeArg<tensor<scalar_t, tensor_sizes...>>
|
||||
{
|
||||
using scalar_type = std::remove_const_t<scalar_t>;
|
||||
using type = std::conditional_t<
|
||||
sizeof...(tensor_sizes) == 0,
|
||||
scalar_type,
|
||||
tensor<scalar_type, tensor_sizes...>>&;
|
||||
};
|
||||
|
||||
template <typename qf_param_ts>
|
||||
struct LocalQFShapeFunction;
|
||||
|
||||
template <typename... qf_param_ts>
|
||||
struct LocalQFShapeFunction<tuple<qf_param_ts...>>
|
||||
{
|
||||
void operator()(
|
||||
typename LocalQFShapeArg<qf_param_decay_t<qf_param_ts>>::type...) const;
|
||||
};
|
||||
|
||||
template <typename qfunc_t>
|
||||
using LocalQFShapeFunctionFor = LocalQFShapeFunction<
|
||||
typename get_function_signature<qfunc_t>::type::parameter_ts>;
|
||||
|
||||
} // namespace detail
|
||||
|
||||
struct GlobalQFBackend
|
||||
{
|
||||
/**
|
||||
* @brief Make an action for a global Q-function.
|
||||
*
|
||||
* @param ctx The integrator context.
|
||||
* @param args The arguments to the action.
|
||||
* @return The action.
|
||||
*/
|
||||
template<
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
auto static MakeAction(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs)
|
||||
{
|
||||
return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Make a derivative action for a global Q-function.
|
||||
*
|
||||
* @tparam derivative_id The id of the derivative.
|
||||
* @param ctx The integrator context.
|
||||
* @param args The arguments to the derivative action.
|
||||
* @return The derivative action.
|
||||
*/
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
auto static MakeDerivativeAction(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs)
|
||||
{
|
||||
return GlobalQFImpl::DerivativeAction<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(
|
||||
ctx, qfunc, inputs, outputs);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Make a derivative setup for a global Q-function.
|
||||
*
|
||||
* @tparam derivative_id The id of the derivative.
|
||||
* @param ctx The integrator context.
|
||||
* @param args The arguments to the derivative setup.
|
||||
* @return The derivative setup.
|
||||
*/
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
auto static MakeDerivativeSetup(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
Vector &qp_cache)
|
||||
{
|
||||
return GlobalQFImpl::DerivativeSetup<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(
|
||||
ctx, qfunc, inputs, outputs, qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
auto static MakeDerivativeApply(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t & /*qfunc*/,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeApply<
|
||||
derivative_id,
|
||||
detail::LocalQFShapeFunctionFor<qfunc_t>,
|
||||
inputs_t,
|
||||
outputs_t>(ctx,
|
||||
detail::LocalQFShapeFunctionFor<qfunc_t> {},
|
||||
inputs,
|
||||
outputs,
|
||||
qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
auto static MakeDerivativeApplyTranspose(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return GlobalQFImpl::DerivativeApplyTranspose<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(
|
||||
ctx, qfunc, inputs, outputs, qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
auto static MakeDerivativeAssemble(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeAssemble<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(
|
||||
ctx, qfunc, inputs, outputs, qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
auto static MakeDerivativeAssembleDiagonal(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeAssembleDiagonal<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(
|
||||
ctx, qfunc, inputs, outputs, qp_cache);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mfem::future
|
||||
@@ -1,658 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../fieldoperator.hpp"
|
||||
#include "kernels.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
#include "../util.hpp"
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace mfem::future::LocalQFImpl
|
||||
{
|
||||
|
||||
template <typename outputs_t, typename index_seq>
|
||||
struct action_outputs_direct_impl;
|
||||
|
||||
template <typename outputs_t, std::size_t... Is>
|
||||
struct action_outputs_direct_impl<outputs_t, std::index_sequence<Is...>>
|
||||
{
|
||||
static constexpr bool value =
|
||||
((is_identity_fop_v<tuple_element_t<Is, outputs_t>> ||
|
||||
is_functionalvalue_fop_v<tuple_element_t<Is, outputs_t>>) && ...);
|
||||
};
|
||||
|
||||
template <typename outputs_t>
|
||||
constexpr bool action_outputs_direct_v = action_outputs_direct_impl<outputs_t,
|
||||
std::make_index_sequence<tuple_size<outputs_t>::value>>::value;
|
||||
|
||||
template <typename qfunc_t, typename inputs_t, typename outputs_t,
|
||||
typename index_seq>
|
||||
struct action_outputs_direct_value_impl;
|
||||
|
||||
template <typename qfunc_t, typename inputs_t, typename outputs_t,
|
||||
std::size_t... Is>
|
||||
struct action_outputs_direct_value_impl<qfunc_t, inputs_t, outputs_t,
|
||||
std::index_sequence<Is...>>
|
||||
{
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr bool value =
|
||||
((is_identity_fop_v<tuple_element_t<Is, outputs_t>> ||
|
||||
is_functionalvalue_fop_v<tuple_element_t<Is, outputs_t>>) && ...)
|
||||
&& ((!qf_param_uses_dual_v<
|
||||
typename qf_param_slot<qfunc_t, n_inputs + Is>::qf_decay_param_t>)
|
||||
&& ...);
|
||||
};
|
||||
|
||||
template <typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
constexpr bool action_outputs_direct_value_v =
|
||||
action_outputs_direct_value_impl<qfunc_t, inputs_t, outputs_t,
|
||||
std::make_index_sequence<tuple_size<outputs_t>::value>>::value;
|
||||
|
||||
template<typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
class Action
|
||||
{
|
||||
static constexpr auto inout_tuple =
|
||||
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
|
||||
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
|
||||
static constexpr size_t nfields =
|
||||
count_unique_field_ids(filtered_inout_tuple);
|
||||
|
||||
using qf_signature = typename get_function_signature<qfunc_t>::type;
|
||||
using qf_param_ts = typename qf_signature::parameter_ts;
|
||||
using args_tuple_t = decay_tuple<qf_param_ts>;
|
||||
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
|
||||
"LocalQF: q-function arity must match inputs + outputs");
|
||||
|
||||
template<typename backend_t, std::size_t I, typename RArgs, typename InXEs>
|
||||
static MFEM_HOST_DEVICE decltype(auto) direct_input_arg(
|
||||
RArgs &rargs,
|
||||
const InXEs &in_XE,
|
||||
const int qx,
|
||||
const int qy,
|
||||
const int qz,
|
||||
const int e)
|
||||
{
|
||||
const auto &XE = in_XE[I];
|
||||
using FOP = tuple_element_t<I, inputs_t>;
|
||||
using ARG = typename qf_param_slot<qfunc_t, I>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
using DT = typename qf_param_slot<qfunc_t, I>::qf_decay_param_t;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
return backend_t::template identity_qp_pull_dual<DT>(
|
||||
false, XE, XE, qx, qy, qz, e);
|
||||
}
|
||||
else
|
||||
{
|
||||
return as_tensor<ARG>(&XE(0, qx, qy, qz, e));
|
||||
}
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
return XE(qx, qy, qz, 0, 0);
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
|
||||
{
|
||||
return backend_t::template qp_pull<ARG>(get<I>(rargs), qx, qy, qz);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
}
|
||||
|
||||
template<std::size_t I, typename OutYEs>
|
||||
static MFEM_HOST_DEVICE decltype(auto) direct_output_arg(
|
||||
const OutYEs &out_YE,
|
||||
const int qx,
|
||||
const int qy,
|
||||
const int qz,
|
||||
const int e)
|
||||
{
|
||||
constexpr size_t o = n_inputs + I;
|
||||
const auto &YE = out_YE[I];
|
||||
using DT = typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
|
||||
using ARG = typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
|
||||
if constexpr (std::is_same_v<DT, real_t>)
|
||||
{
|
||||
return YE(0, qx, qy, qz, e);
|
||||
}
|
||||
else
|
||||
{
|
||||
return as_tensor<ARG>(&YE(0, qx, qy, qz, e));
|
||||
}
|
||||
}
|
||||
|
||||
template<typename backend_t, typename RArgs, typename InXEs, typename OutYEs,
|
||||
std::size_t... InIs, std::size_t... OutIs>
|
||||
static MFEM_HOST_DEVICE void call_qfunc_direct(
|
||||
const qfunc_t &qfunc,
|
||||
RArgs &rargs,
|
||||
const InXEs &in_XE,
|
||||
const OutYEs &out_YE,
|
||||
const int qx,
|
||||
const int qy,
|
||||
const int qz,
|
||||
const int e,
|
||||
std::index_sequence<InIs...>,
|
||||
std::index_sequence<OutIs...>)
|
||||
{
|
||||
qfunc(direct_input_arg<backend_t, InIs>(rargs, in_XE, qx, qy, qz, e)...,
|
||||
direct_output_arg<OutIs>(out_YE, qx, qy, qz, e)...);
|
||||
}
|
||||
|
||||
const qfunc_t qfunc;
|
||||
const inputs_t inputs;
|
||||
const outputs_t outputs;
|
||||
const IntegratorContext ctx;
|
||||
const std::vector<const DofToQuad *> dtqs;
|
||||
// inputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
const std::array<DofToQuadMap, n_inputs> input_dtq;
|
||||
const std::array<size_t, n_inputs> input_idx; // input to field
|
||||
const std::array<const real_t *, n_inputs> input_B, input_G;
|
||||
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
|
||||
// outputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
const std::array<DofToQuadMap, n_outputs> output_dtq;
|
||||
const std::array<size_t, n_outputs> output_idx; // output to field
|
||||
const std::array<const real_t *, n_outputs> output_B, output_G;
|
||||
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
|
||||
// other constants
|
||||
const int dim, ne, nq, q1d;
|
||||
|
||||
public:
|
||||
////////////////////////////////////////////////////////
|
||||
Action() = delete;
|
||||
|
||||
Action(IntegratorContext ctx,
|
||||
qfunc_t qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs):
|
||||
qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
|
||||
dtqs(make_dtqs(ctx)),
|
||||
// inputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
input_dtq(create_dtq_maps<Entity::Element>(
|
||||
inputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, inputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
input_idx(create_input_vector_map(ctx, inputs)),
|
||||
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
|
||||
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
|
||||
input_vdim(get_vdim(inputs)),
|
||||
// outputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
output_dtq(create_dtq_maps<Entity::Element>(
|
||||
outputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, outputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
output_idx(create_output_vector_map(ctx, outputs)),
|
||||
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
|
||||
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
|
||||
output_vdim(get_vdim(outputs)),
|
||||
// other constants
|
||||
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
|
||||
q1d(tensor_1d_size(nq, dim))
|
||||
{
|
||||
MFEM_ASSERT(ctx.unionfds.size() == nfields,
|
||||
"LocalQFBackend: unionfds size mismatch");
|
||||
}
|
||||
|
||||
template<typename Backend>
|
||||
void run_kernels(const std::vector<Vector *> &xe,
|
||||
std::vector<Vector *> &ye) const
|
||||
{
|
||||
Backend::Run(dim,
|
||||
q1d,
|
||||
// arguments
|
||||
ctx,
|
||||
qfunc,
|
||||
// inputs
|
||||
input_idx,
|
||||
input_B,
|
||||
input_G,
|
||||
input_vdim,
|
||||
input_d1d,
|
||||
input_q1d,
|
||||
// outputs
|
||||
output_idx,
|
||||
output_B,
|
||||
output_G,
|
||||
output_vdim,
|
||||
output_d1d,
|
||||
output_q1d,
|
||||
// input and output vectors
|
||||
xe,
|
||||
ye,
|
||||
// fallback arguments
|
||||
dim,
|
||||
q1d);
|
||||
}
|
||||
|
||||
void operator()(const std::vector<Vector *> &xe,
|
||||
std::vector<Vector *> &ye) const
|
||||
{
|
||||
if (q1d <= LocalQFLOBackendMQ1())
|
||||
{
|
||||
run_kernels<ActionLO>(xe, ye);
|
||||
}
|
||||
else if (q1d <= LocalQFHOBackendMQ1())
|
||||
{
|
||||
run_kernels<ActionHO>(xe, ye);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
|
||||
static void
|
||||
action_callback(const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
// inputs: idx, B, G, vdim, d1d, q1d
|
||||
const std::array<size_t, n_inputs> &in_idx,
|
||||
const std::array<const real_t *, n_inputs> in_B,
|
||||
const std::array<const real_t *, n_inputs> in_G,
|
||||
const std::array<int, n_inputs> &in_vdim,
|
||||
const std::array<int, n_inputs> &in_d1d,
|
||||
const std::array<int, n_inputs> &in_q1d,
|
||||
// outputs: idx, B, G, vdim, d1d, q1d
|
||||
const std::array<size_t, n_outputs> &out_idx,
|
||||
const std::array<const real_t *, n_outputs> out_B,
|
||||
const std::array<const real_t *, n_outputs> out_G,
|
||||
const std::array<int, n_outputs> &out_vdim,
|
||||
const std::array<int, n_outputs> &out_d1d,
|
||||
const std::array<int, n_outputs> &out_q1d,
|
||||
const std::vector<Vector *> &xe,
|
||||
std::vector<Vector *> &ye,
|
||||
// fallback arguments
|
||||
const int dim,
|
||||
const int q1d)
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
MFEM_CONTRACT_VAR(dim);
|
||||
MFEM_ASSERT(dim == ctx.mesh.Dimension(), "Dimension mismatch");
|
||||
|
||||
static constexpr auto B2D = backend_t::DIM == 2;
|
||||
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::Q1D;
|
||||
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
|
||||
|
||||
const int ne = ctx.nentities;
|
||||
|
||||
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
|
||||
|
||||
// --------------------------------------------------
|
||||
// INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
|
||||
// --------------------------------------------------
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const size_t k = in_idx[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
|
||||
in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
|
||||
in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
|
||||
"tensor-product IR expected");
|
||||
in_XE[i] = Reshape(
|
||||
ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// --------------------------------------------------
|
||||
// OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
|
||||
// --------------------------------------------------
|
||||
std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const size_t k = out_idx[i];
|
||||
const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
|
||||
{
|
||||
MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
|
||||
out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP> || is_functionalvalue_fop_v<FOP>)
|
||||
{
|
||||
MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
|
||||
out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported FieldOperator");
|
||||
}
|
||||
});
|
||||
|
||||
const auto d_attr = ctx.attr.Read();
|
||||
const bool has_attr = ctx.attr.Size() > 0;
|
||||
const auto d_elem_attr = ctx.elem_attr->Read();
|
||||
|
||||
dfem::forall<MTPB>(
|
||||
[=] MFEM_HOST_DEVICE(const int e, void *)
|
||||
{
|
||||
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
|
||||
|
||||
// -----------------------------------------------
|
||||
// Inputs and outputs argument registers
|
||||
// -----------------------------------------------
|
||||
action_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Shared memory
|
||||
// -----------------------------------------------
|
||||
MFEM_SHARED typename backend_t::Shared smem;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Load inputs
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const auto &XE = in_XE[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
|
||||
;
|
||||
const real_t *B = in_B[i], *G = in_G[i];
|
||||
auto &rarg = get<i>(rargs);
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop<FOP>::value)
|
||||
{
|
||||
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
using XE_t = decltype(XE);
|
||||
using rarg_t = decltype(rarg);
|
||||
using qf_param_t =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
|
||||
backend_t::template LoadGradient<RNK, rarg_t, XE_t, qf_param_t>(
|
||||
smem, e, d, q, q1d, B, G, XE, rarg);
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
|
||||
{
|
||||
// qp values are read directly from in_XE / IR
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------
|
||||
// Evaluate the quadrature function
|
||||
// Warning: no 'DIRECT' on the 'Z' direction,
|
||||
// as one backend may need to iterate over it.
|
||||
// -----------------------------------------------
|
||||
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
if constexpr (action_outputs_direct_value_v<qfunc_t, inputs_t,
|
||||
outputs_t>)
|
||||
{
|
||||
call_qfunc_direct<backend_t>(
|
||||
qfunc, rargs, in_XE, out_YE, qx, qy, qz, e,
|
||||
std::make_index_sequence<n_inputs> {},
|
||||
std::make_index_sequence<n_outputs> {});
|
||||
}
|
||||
else
|
||||
{
|
||||
args_tuple_t qargs;
|
||||
|
||||
// --------------------------------------
|
||||
// Pulling arguments from registers to qargs tuple
|
||||
// --------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
auto &qarg = get<i>(qargs);
|
||||
const auto &XE = in_XE[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
using DT =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
qarg = backend_t::template identity_qp_pull_dual<DT>(
|
||||
false, XE, XE, qx, qy, qz, e);
|
||||
}
|
||||
else
|
||||
{
|
||||
qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
|
||||
}
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
qarg = XE(qx, qy, qz, 0, 0);
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
qarg = backend_t::template qp_pull<ARG>(
|
||||
get<i>(rargs), qx, qy, qz);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// --------------------------------------
|
||||
// Call the quadrature function
|
||||
// --------------------------------------
|
||||
call_qfunc_no_move(qfunc, qargs);
|
||||
|
||||
// --------------------------------------
|
||||
// Pushing arguments from qargs tuple to registers
|
||||
// --------------------------------------
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value, o = n_inputs + i;
|
||||
const auto &qarg = get<o>(qargs);
|
||||
const auto &YE = out_YE[i];
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP> ||
|
||||
is_functionalvalue_fop_v<FOP>)
|
||||
{
|
||||
using DT =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
backend_t::identity_qp_write_value(
|
||||
YE, qx, qy, qz, e, qarg);
|
||||
}
|
||||
else
|
||||
{
|
||||
as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
|
||||
}
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
auto &rarg = get<o>(rargs);
|
||||
backend_t::template qp_push<ARG>(
|
||||
rarg, qx, qy, qz, qarg);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if constexpr (!action_outputs_direct_v<outputs_t>)
|
||||
{
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
|
||||
// -----------------------------------------------
|
||||
// Integrate outputs
|
||||
// -----------------------------------------------
|
||||
if constexpr (!action_outputs_direct_v<outputs_t>)
|
||||
{
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value, o = n_inputs + i;
|
||||
const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
|
||||
const auto B = out_B[i], G = out_G[i];
|
||||
const auto &YE = out_YE[i];
|
||||
auto &rarg = get<o>(rargs);
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP>)
|
||||
{
|
||||
backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
using YE_t = decltype(YE);
|
||||
using rarg_t = decltype(rarg);
|
||||
// Both the rank and the extents have to come from the output
|
||||
// parameter slot o, not from the output index i.
|
||||
using qf_param_t =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
|
||||
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
|
||||
smem, e, d, q, Q1D, B, G, YE, rarg);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP> ||
|
||||
is_functionalvalue_fop_v<FOP>)
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
ne,
|
||||
backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
|
||||
q1d, in_d1d, out_d1d)),
|
||||
0,
|
||||
nullptr);
|
||||
}
|
||||
using KernelType = decltype(&Action::action_callback<>);
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionLO, KernelType, (int, int) );
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionHO, KernelType, (int, int) );
|
||||
};
|
||||
|
||||
// Low Order kernels
|
||||
template<typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
|
||||
Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Kernel()
|
||||
{
|
||||
static_assert(Q1D <= LocalQFLOBackend<DIM>::MQ1);
|
||||
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
|
||||
if constexpr (DIM == 3 && Q1D == LocalQFLOBackendMQ1() &&
|
||||
action_outputs_direct_value_v<qfunc_t, inputs_t, outputs_t>)
|
||||
{
|
||||
return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D, Q1D / 2>>;
|
||||
}
|
||||
else
|
||||
{
|
||||
return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D>>;
|
||||
}
|
||||
}
|
||||
|
||||
// Low Order fallback
|
||||
template<typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
|
||||
Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
|
||||
using ActionLO = typename action_t::ActionLO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<ActionLO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<ActionLO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
// High Order kernels
|
||||
template<typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
|
||||
Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Kernel()
|
||||
{
|
||||
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
|
||||
return action_t::template action_callback<LocalQFHOBackend<DIM>, Q1D>;
|
||||
}
|
||||
|
||||
// High Order fallback
|
||||
template<typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
|
||||
Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
|
||||
using ActionHO = typename action_t::ActionHO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<ActionHO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<ActionHO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future::LocalQFImpl
|
||||
@@ -1,926 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
|
||||
#include "kernels.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
#include "../util.hpp"
|
||||
|
||||
namespace mfem::future::LocalQFImpl
|
||||
{
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
class DerivativeAction
|
||||
{
|
||||
static constexpr auto inout_tuple =
|
||||
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
|
||||
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
|
||||
static constexpr size_t nfields =
|
||||
count_unique_field_ids(filtered_inout_tuple);
|
||||
|
||||
using qf_signature = typename get_function_signature<qfunc_t>::type;
|
||||
using qf_param_ts = typename qf_signature::parameter_ts;
|
||||
using args_tuple_t = decay_tuple<qf_param_ts>;
|
||||
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
|
||||
"LocalQF: q-function arity must match inputs + outputs");
|
||||
|
||||
/// Which inputs carry a tangent, i.e. are attached to the field being
|
||||
/// differentiated against. This is a property of `inputs_t` and
|
||||
/// `derivative_id` alone, so it is available at compile time: it decides the
|
||||
/// Enzyme activity of every q-function parameter, which loads the tangent
|
||||
/// pass has to do, and how large the shadow register bank has to be. The
|
||||
/// runtime `input_is_dependent` below holds the same values and is kept for
|
||||
/// the host-side sizing checks.
|
||||
static constexpr auto input_activity =
|
||||
mfem::future::detail::make_activity_map <
|
||||
static_cast<std::size_t>(derivative_id) > (inputs_t {});
|
||||
static_assert(input_activity.size() == n_inputs);
|
||||
|
||||
/// Shadow register bank: only the active input slots are materialized.
|
||||
template <typename backend_t, int MQ1, std::size_t... Is>
|
||||
static auto shadow_bank_type(std::index_sequence<Is...>)
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
-> masked_input_args_reg_t<backend_t, qfunc_t, MQ1, input_activity[Is]...>;
|
||||
#else
|
||||
// The dual-number path pulls through every input slot unconditionally.
|
||||
-> input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1>;
|
||||
#endif
|
||||
|
||||
template <typename backend_t, int MQ1>
|
||||
using shadow_bank_t = decltype(shadow_bank_type<backend_t, MQ1>(
|
||||
std::make_index_sequence<n_inputs> {}));
|
||||
|
||||
/// Per-quadrature-point shadow argument tuple. Only the active inputs and
|
||||
/// the outputs are materialized where supported; the rest are `enzyme_const`
|
||||
/// and their shadow slots are never addressed. This is the innermost live
|
||||
/// state of the kernel, and on device it shares a per-thread register budget
|
||||
/// capped by the launch bounds, so the dead slots are worth removing
|
||||
/// explicitly rather than hoping the optimizer splits the tuple.
|
||||
#ifdef _MSC_VER
|
||||
using shadow_args_t = args_tuple_t;
|
||||
#else
|
||||
template <std::size_t... Is>
|
||||
static auto shadow_tuple_type(std::index_sequence<Is...>)
|
||||
-> masked_args_tuple_t < args_tuple_t,
|
||||
(Is<n_inputs ? input_activity[Is] : true)... >;
|
||||
|
||||
using shadow_args_t = decltype(shadow_tuple_type(
|
||||
std::make_index_sequence<n_inputs + n_outputs> {}));
|
||||
#endif
|
||||
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
/// Forward-mode call with the activity of every q-function parameter fixed
|
||||
/// at compile time. Outputs are always active; inputs follow
|
||||
/// `input_activity`, so an inactive input (the mesh nodes and the quadrature
|
||||
/// weight, for a derivative w.r.t. the trial field) is marked `enzyme_const`
|
||||
/// rather than dup'd with a zero tangent. Without this Enzyme differentiates
|
||||
/// everything those inputs feed - for a diffusion q-function the whole
|
||||
/// inv(J) / det(J) chain - to produce a tangent that is structurally zero.
|
||||
// `qf_t` is deduced because the kernel captures the q-function by value into
|
||||
// a const lambda, so it arrives here as `const qfunc_t`.
|
||||
template <typename qf_t, std::size_t... Is>
|
||||
MFEM_FUTURE_ALWAYS_INLINE
|
||||
MFEM_HOST_DEVICE static void call_fwddiff(qf_t &qfunc,
|
||||
args_tuple_t &primal_args,
|
||||
shadow_args_t &shadow_args,
|
||||
std::index_sequence<Is...>)
|
||||
{
|
||||
mfem::future::call_enzyme_fwddiff_active <
|
||||
(Is < n_inputs ? input_activity[Is] : true)... > (
|
||||
qfunc, primal_args, shadow_args);
|
||||
}
|
||||
#endif
|
||||
|
||||
qfunc_t qfunc;
|
||||
const inputs_t inputs;
|
||||
const outputs_t outputs;
|
||||
const IntegratorContext ctx;
|
||||
const std::vector<const DofToQuad *> dtqs;
|
||||
// inputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
const std::array<DofToQuadMap, n_inputs> input_dtq;
|
||||
const std::array<size_t, n_inputs> input_idx;
|
||||
const std::array<const real_t *, n_inputs> input_B, input_G;
|
||||
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
|
||||
// outputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
const std::array<DofToQuadMap, n_outputs> output_dtq;
|
||||
const std::array<size_t, n_outputs> output_idx;
|
||||
const std::array<const real_t *, n_outputs> output_B, output_G;
|
||||
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
|
||||
// other constants
|
||||
const int dim, ne, nq, q1d;
|
||||
|
||||
std::array<bool, n_inputs> input_is_dependent;
|
||||
FieldDescriptor direction_fd;
|
||||
mutable Vector direction_e;
|
||||
mutable RestrictionCache<Entity::Element> direction_rcache;
|
||||
|
||||
public:
|
||||
//////////////////////////////////////////////////////////////////
|
||||
DerivativeAction() = delete;
|
||||
|
||||
DerivativeAction(IntegratorContext ctx,
|
||||
qfunc_t qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs):
|
||||
qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
|
||||
dtqs(make_dtqs(ctx)),
|
||||
// inputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
input_dtq(create_dtq_maps<Entity::Element>(
|
||||
inputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, inputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
input_idx(create_input_vector_map(ctx, inputs)),
|
||||
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
|
||||
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
|
||||
input_vdim(get_vdim(inputs)),
|
||||
// outputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
output_dtq(create_dtq_maps<Entity::Element>(
|
||||
outputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, outputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
output_idx(create_output_vector_map(ctx, outputs)),
|
||||
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
|
||||
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
|
||||
output_vdim(get_vdim(outputs)),
|
||||
// other constants
|
||||
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
|
||||
q1d(tensor_1d_size(nq, dim))
|
||||
{
|
||||
MFEM_ASSERT(ctx.unionfds.size() == nfields,
|
||||
"LocalQFBackend: unionfds size mismatch");
|
||||
|
||||
// Determine which inputs are dependent on the derivative direction
|
||||
auto dependency_map = make_dependency_map(inputs);
|
||||
auto it = dependency_map.find(derivative_id);
|
||||
MFEM_ASSERT(it != dependency_map.end(),
|
||||
"Derivative ID not found in dependency map");
|
||||
input_is_dependent = it->second;
|
||||
|
||||
// Find direction field index
|
||||
int direction_field_idx = -1;
|
||||
for (size_t uf = 0; uf < nfields; uf++)
|
||||
{
|
||||
if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
|
||||
{
|
||||
direction_field_idx = static_cast<int>(uf);
|
||||
break;
|
||||
}
|
||||
}
|
||||
MFEM_ASSERT(
|
||||
direction_field_idx != -1,
|
||||
"LocalQFBackend: derivative direction field not found in unionfds");
|
||||
direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename Backend>
|
||||
void run_kernels(const std::vector<Vector *> &xe,
|
||||
std::vector<Vector *> &ye)
|
||||
{
|
||||
Backend::Run(dim,
|
||||
q1d,
|
||||
// arguments
|
||||
ctx,
|
||||
qfunc,
|
||||
// inputs
|
||||
input_idx,
|
||||
input_B,
|
||||
input_G,
|
||||
input_vdim,
|
||||
input_d1d,
|
||||
input_q1d,
|
||||
// outputs
|
||||
output_idx,
|
||||
output_B,
|
||||
output_G,
|
||||
output_vdim,
|
||||
output_d1d,
|
||||
output_q1d,
|
||||
// input and output vectors
|
||||
xe,
|
||||
ye,
|
||||
input_is_dependent,
|
||||
direction_e,
|
||||
// fallback arguments
|
||||
dim,
|
||||
q1d);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
void operator()(const std::vector<Vector *> &xe,
|
||||
const Vector *direction_l,
|
||||
std::vector<Vector *> &ye)
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
MFEM_ASSERT(direction_l != nullptr,
|
||||
"LocalQF DerivativeAction: direction vector is null");
|
||||
|
||||
restriction(direction_fd,
|
||||
direction_rcache,
|
||||
*direction_l,
|
||||
direction_e,
|
||||
ElementDofOrdering::LEXICOGRAPHIC);
|
||||
if (q1d <= LocalQFLOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeActionLO>(xe, ye);
|
||||
}
|
||||
else if (q1d <= LocalQFHOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeActionHO>(xe, ye);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template<typename backend_t, int T_Q1D>
|
||||
struct DerivativeActionKernelData
|
||||
{
|
||||
qfunc_t qfunc;
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
|
||||
std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
|
||||
std::array<const real_t *, n_inputs> in_B;
|
||||
std::array<const real_t *, n_inputs> in_G;
|
||||
std::array<int, n_inputs> in_d1d;
|
||||
std::array<int, n_inputs> in_q1d;
|
||||
std::array<const real_t *, n_outputs> out_B;
|
||||
std::array<const real_t *, n_outputs> out_G;
|
||||
std::array<int, n_outputs> out_d1d;
|
||||
std::array<int, n_outputs> out_q1d;
|
||||
std::array<bool, n_inputs> input_dep;
|
||||
const int *d_attr;
|
||||
bool has_attr;
|
||||
const int *d_elem_attr;
|
||||
int q1d;
|
||||
};
|
||||
|
||||
template<typename backend_t, int T_Q1D>
|
||||
MFEM_FUTURE_ALWAYS_INLINE MFEM_HOST_DEVICE static void
|
||||
derivative_action_kernel_body(
|
||||
const DerivativeActionKernelData<backend_t, T_Q1D> &data, const int e)
|
||||
{
|
||||
static constexpr auto B2D = backend_t::DIM == 2;
|
||||
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
|
||||
auto &qfunc = data.qfunc;
|
||||
const auto &in_XE = data.in_XE;
|
||||
const auto &in_XE_dir = data.in_XE_dir;
|
||||
const auto &out_YE = data.out_YE;
|
||||
const auto &in_B = data.in_B;
|
||||
const auto &in_G = data.in_G;
|
||||
const auto &in_d1d = data.in_d1d;
|
||||
const auto &in_q1d = data.in_q1d;
|
||||
const auto &out_B = data.out_B;
|
||||
const auto &out_G = data.out_G;
|
||||
const auto &out_d1d = data.out_d1d;
|
||||
const auto &out_q1d = data.out_q1d;
|
||||
const auto &input_dep = data.input_dep;
|
||||
MFEM_CONTRACT_VAR(input_dep);
|
||||
const auto d_attr = data.d_attr;
|
||||
const bool has_attr = data.has_attr;
|
||||
const auto d_elem_attr = data.d_elem_attr;
|
||||
const int q1d = data.q1d;
|
||||
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
|
||||
|
||||
// -----------------------------------------------
|
||||
// Inputs and outputs argument registers
|
||||
// -----------------------------------------------
|
||||
args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
|
||||
shadow_bank_t<backend_t, MQ1> sargs; // shadow, active inputs only
|
||||
|
||||
// -----------------------------------------------
|
||||
// Shared memory
|
||||
// -----------------------------------------------
|
||||
MFEM_SHARED typename backend_t::Shared smem;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Load primal inputs (rargs)
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const auto &XE = in_XE[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
|
||||
const real_t *B = in_B[i], *G = in_G[i];
|
||||
auto &rarg = get<i>(rargs);
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop<FOP>::value)
|
||||
{
|
||||
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
|
||||
using FieldParamT =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
backend_t::template LoadGradient<RNK,
|
||||
decltype(rarg),
|
||||
decltype(XE),
|
||||
FieldParamT>(
|
||||
smem, e, d, q, Q1D, B, G, XE, rarg);
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
|
||||
{
|
||||
// qp values are read directly from in_XE / IR
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------
|
||||
// Load tangent directions (sargs)
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (input_activity[i] &&
|
||||
(is_value_fop_v<FOP> || is_gradient_fop_v<FOP>))
|
||||
{
|
||||
const auto &XE = in_XE_dir[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
|
||||
const real_t *B = in_B[i], *G = in_G[i];
|
||||
auto &sarg = get<i>(sargs); // shadow argument register
|
||||
if constexpr (is_value_fop_v<FOP>)
|
||||
{
|
||||
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
|
||||
using FieldParamT =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
backend_t::template LoadGradient<RNK,
|
||||
decltype(sarg),
|
||||
decltype(XE),
|
||||
FieldParamT>(
|
||||
smem, e, d, q, Q1D, B, G, XE, sarg);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Inactive input, or an input read straight from quadrature
|
||||
// point data (weight / identity): nothing to interpolate.
|
||||
static_assert(!input_activity[i] || is_weight_fop_v<FOP> ||
|
||||
is_identity_fop_v<FOP>, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------
|
||||
// Evaluate the quadrature function
|
||||
// Warning: no 'DIRECT' on the 'Z' direction,
|
||||
// as one backend may need to iterate over it.
|
||||
// -----------------------------------------------
|
||||
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
args_tuple_t primal_args {};
|
||||
shadow_args_t shadow_args {};
|
||||
|
||||
// --------------------------------------
|
||||
// Pulling arguments from registers to primal and shadow
|
||||
// tuples
|
||||
// --------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
auto &parg = get<i>(primal_args);
|
||||
auto &targ = get<i>(shadow_args);
|
||||
const auto &XE = in_XE[i];
|
||||
const auto &XEd = in_XE_dir[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
|
||||
MFEM_CONTRACT_VAR(targ);
|
||||
MFEM_CONTRACT_VAR(XEd);
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
|
||||
if constexpr (input_activity[i])
|
||||
{
|
||||
targ = as_tensor<ARG>(&XEd(0, qx, qy, qz, e));
|
||||
}
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
parg = XE(qx, qy, qz, 0, 0);
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
parg = backend_t::template qp_pull<ARG>(
|
||||
get<i>(rargs), qx, qy, qz);
|
||||
if constexpr (input_activity[i])
|
||||
{
|
||||
targ = backend_t::template qp_pull<ARG>(
|
||||
get<i>(sargs), qx, qy, qz);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// --------------------------------------
|
||||
// Call the quadrature function. Inactive inputs are
|
||||
// enzyme_const, so their shadow slots are never read and are
|
||||
// deliberately left unset above.
|
||||
// --------------------------------------
|
||||
call_fwddiff(qfunc, primal_args, shadow_args,
|
||||
std::make_index_sequence<n_inputs + n_outputs> {});
|
||||
|
||||
// --------------------------------------
|
||||
// Pushing arguments from enzyme_shadow tuple to registers
|
||||
// --------------------------------------
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value, o = n_inputs + i;
|
||||
const auto &qout = get<o>(shadow_args);
|
||||
auto &YE = out_YE[i];
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qout;
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
auto &rarg = get<o>(rargs);
|
||||
backend_t::template qp_push_tangent<ARG>(
|
||||
rarg, qx, qy, qz, qout);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
#else // MFEM_USE_ENZYME
|
||||
args_tuple_t qargs;
|
||||
|
||||
// --------------------------------------
|
||||
// Pulling arguments from registers to qargs tuple
|
||||
// --------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
auto &qarg = get<i>(qargs);
|
||||
const auto &XE = in_XE[i];
|
||||
const auto &XEd = in_XE_dir[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
using DT =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
qarg = backend_t::template identity_qp_pull_dual<DT>(
|
||||
input_dep[i], XE, XEd, qx, qy, qz, e);
|
||||
}
|
||||
else
|
||||
{
|
||||
qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
|
||||
}
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
qarg = XE(qx, qy, qz, 0, 0);
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
qarg = backend_t::template qp_pull_directional<ARG>(
|
||||
get<i>(rargs),
|
||||
get<i>(sargs),
|
||||
qx,
|
||||
qy,
|
||||
qz,
|
||||
input_dep[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// --------------------------------------
|
||||
// Call the quadrature function
|
||||
// --------------------------------------
|
||||
call_qfunc_no_move(qfunc, qargs);
|
||||
|
||||
// --------------------------------------
|
||||
// Pushing arguments from qargs tuple to registers
|
||||
// --------------------------------------
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value, o = n_inputs + i;
|
||||
const auto &qarg = get<o>(qargs);
|
||||
const auto &YE = out_YE[i];
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
using DT =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
backend_t::identity_qp_write_tangent(
|
||||
YE, qx, qy, qz, e, qarg);
|
||||
}
|
||||
else
|
||||
{
|
||||
as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
|
||||
}
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
auto &rarg = get<o>(rargs);
|
||||
backend_t::template qp_push_tangent<ARG>(
|
||||
rarg, qx, qy, qz, qarg);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
#endif // MFEM_USE_ENZYME
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Integrate outputs
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value, o = n_inputs + i;
|
||||
const int d = out_d1d[i], q = out_q1d[i];
|
||||
const auto B = out_B[i], G = out_G[i];
|
||||
auto &YE = out_YE[i];
|
||||
auto &rarg = get<o>(rargs);
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP>)
|
||||
{
|
||||
backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
using YE_t = decltype(YE);
|
||||
using rarg_t = decltype(rarg);
|
||||
using qf_param_t =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
|
||||
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
|
||||
smem, e, d, q, q1d, B, G, YE, rarg);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
template<typename backend_t, int T_Q1D>
|
||||
struct DerivativeActionKernelBody
|
||||
{
|
||||
MFEM_FUTURE_ALWAYS_INLINE MFEM_HOST_DEVICE static void run(
|
||||
const DerivativeActionKernelData<backend_t, T_Q1D> &data, const int e)
|
||||
{
|
||||
derivative_action_kernel_body<backend_t, T_Q1D>(data, e);
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
|
||||
static void
|
||||
derivative_action_callback(const IntegratorContext &ctx,
|
||||
qfunc_t &qfunc,
|
||||
// inputs: idx, B, G, vdim, d1d, q1d
|
||||
const std::array<size_t, n_inputs> &in_idx,
|
||||
const std::array<const real_t *, n_inputs> in_B,
|
||||
const std::array<const real_t *, n_inputs> in_G,
|
||||
const std::array<int, n_inputs> &in_vdim,
|
||||
const std::array<int, n_inputs> &in_d1d,
|
||||
const std::array<int, n_inputs> &in_q1d,
|
||||
// outputs: idx, B, G, vdim, d1d, q1d
|
||||
const std::array<size_t, n_outputs> &out_idx,
|
||||
const std::array<const real_t *, n_outputs> out_B,
|
||||
const std::array<const real_t *, n_outputs> out_G,
|
||||
const std::array<int, n_outputs> &out_vdim,
|
||||
const std::array<int, n_outputs> &out_d1d,
|
||||
const std::array<int, n_outputs> &out_q1d,
|
||||
const std::vector<Vector *> &xe,
|
||||
std::vector<Vector *> &ye,
|
||||
const std::array<bool, n_inputs> &input_dep,
|
||||
const Vector &direction_e,
|
||||
// fallback arguments
|
||||
const int dim,
|
||||
const int q1d)
|
||||
{
|
||||
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
|
||||
// Dependency is resolved at compile time through `input_activity`; the
|
||||
// runtime array is only carried for the non-Enzyme dual-number path.
|
||||
MFEM_CONTRACT_VAR(input_dep);
|
||||
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
static constexpr auto B2D = backend_t::DIM == 2;
|
||||
|
||||
const int ne = ctx.nentities;
|
||||
|
||||
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
|
||||
|
||||
// --------------------------------------------------
|
||||
// INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
|
||||
// --------------------------------------------------
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const size_t k = in_idx[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
|
||||
in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
|
||||
in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
|
||||
"tensor-product IR expected");
|
||||
in_XE[i] = Reshape(
|
||||
ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
const auto d_direction = direction_e.Read();
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const size_t k = in_idx[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
|
||||
{
|
||||
if constexpr (input_activity[i])
|
||||
{
|
||||
MFEM_ASSERT(direction_e.Size() == xe[k]->Size(),
|
||||
"direction E-vector size mismatch for input " << i);
|
||||
in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else
|
||||
{
|
||||
in_XE_dir[i] = in_XE[i];
|
||||
}
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
if constexpr (input_activity[i])
|
||||
{
|
||||
MFEM_VERIFY(direction_e.Size() == xe[k]->Size(),
|
||||
"direction E-vector size mismatch (identity input) "
|
||||
<< i);
|
||||
in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else
|
||||
{
|
||||
in_XE_dir[i] = in_XE[i];
|
||||
}
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>) { in_XE_dir[i] = in_XE[i]; }
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// --------------------------------------------------
|
||||
// OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
|
||||
// --------------------------------------------------
|
||||
std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const size_t k = out_idx[i];
|
||||
const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
|
||||
{
|
||||
MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
|
||||
out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
|
||||
out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported FieldOperator");
|
||||
}
|
||||
});
|
||||
|
||||
const auto d_attr = ctx.attr.Read();
|
||||
const bool has_attr = ctx.attr.Size() > 0;
|
||||
const auto d_elem_attr = ctx.elem_attr->Read();
|
||||
|
||||
DerivativeActionKernelData<backend_t, T_Q1D> data
|
||||
{
|
||||
qfunc,
|
||||
in_XE,
|
||||
in_XE_dir,
|
||||
out_YE,
|
||||
in_B,
|
||||
in_G,
|
||||
in_d1d,
|
||||
in_q1d,
|
||||
out_B,
|
||||
out_G,
|
||||
out_d1d,
|
||||
out_q1d,
|
||||
input_dep,
|
||||
d_attr,
|
||||
has_attr,
|
||||
d_elem_attr,
|
||||
q1d
|
||||
};
|
||||
|
||||
const auto blocks = backend_t::thread_blocks(
|
||||
compute_kernel_thread_1d<inputs_t, outputs_t>(q1d, in_d1d, out_d1d));
|
||||
|
||||
if (Device::Allows(Backend::CUDA_MASK) ||
|
||||
Device::Allows(Backend::HIP_MASK))
|
||||
{
|
||||
dfem::forall_data<backend_t::MAX_THREADS_PER_BLOCK(),
|
||||
DerivativeActionKernelBody<backend_t, T_Q1D>>(
|
||||
data, ne, blocks);
|
||||
}
|
||||
else if (Device::Allows(Backend::CPU_MASK))
|
||||
{
|
||||
for (int e = 0; e < ne; e++)
|
||||
{
|
||||
derivative_action_kernel_body<backend_t, T_Q1D>(data, e);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("no compute backend available");
|
||||
}
|
||||
}
|
||||
|
||||
using DerivativeKernelType =
|
||||
decltype(&DerivativeAction::derivative_action_callback<>);
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionLO,
|
||||
DerivativeKernelType,
|
||||
(int, int) );
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionHO,
|
||||
DerivativeKernelType,
|
||||
(int, int) );
|
||||
};
|
||||
|
||||
// Low Order kernels
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeKernelType
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeActionLO::Kernel()
|
||||
{
|
||||
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
|
||||
using derivative_action_t =
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return derivative_action_t::template derivative_action_callback<
|
||||
LocalQFLOBackend<DIM, Q1D>>;
|
||||
}
|
||||
|
||||
// Low Order fallback
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeKernelType
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeActionLO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using derivative_action_t =
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeActionLO =
|
||||
typename derivative_action_t::DerivativeActionLO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeActionLO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeActionLO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
// High Order kernels
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeKernelType
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeActionHO::Kernel()
|
||||
{
|
||||
using derivative_action_t =
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return derivative_action_t::
|
||||
template derivative_action_callback<LocalQFHOBackend<DIM>, Q1D>;
|
||||
}
|
||||
|
||||
// High Order fallback
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeKernelType
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeActionHO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using derivative_action_t =
|
||||
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeActionHO = typename derivative_action_t::DerivativeActionHO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeActionHO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeActionHO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future::LocalQFImpl
|
||||
@@ -1,738 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
|
||||
#include "kernels.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
#include <array>
|
||||
|
||||
namespace mfem::future::LocalQFImpl
|
||||
{
|
||||
|
||||
// Cached Jacobian apply: J·v from qp_cache filled by DerivativeSetup
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
class DerivativeApply
|
||||
{
|
||||
static constexpr auto inout_tuple =
|
||||
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
|
||||
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
|
||||
static constexpr size_t nfields =
|
||||
count_unique_field_ids(filtered_inout_tuple);
|
||||
|
||||
using qf_signature = typename get_function_signature<qfunc_t>::type;
|
||||
using qf_param_ts = typename qf_signature::parameter_ts;
|
||||
using args_tuple_t = decay_tuple<qf_param_ts>;
|
||||
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
|
||||
"LocalQF: q-function arity must match inputs + outputs");
|
||||
|
||||
const inputs_t inputs;
|
||||
const outputs_t outputs;
|
||||
const IntegratorContext ctx;
|
||||
const Vector &qp_cache;
|
||||
const std::vector<const DofToQuad *> dtqs;
|
||||
// inputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
const std::array<DofToQuadMap, n_inputs> input_dtq;
|
||||
const std::array<size_t, n_inputs> input_idx;
|
||||
const std::array<const real_t *, n_inputs> input_B, input_G;
|
||||
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
|
||||
// outputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
const std::array<DofToQuadMap, n_outputs> output_dtq;
|
||||
const std::array<size_t, n_outputs> output_idx;
|
||||
const std::array<const real_t *, n_outputs> output_B, output_G;
|
||||
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
|
||||
// Jacobian cache metadata
|
||||
const std::array<bool, n_inputs> input_is_dependent;
|
||||
const std::array<int, n_inputs> input_size_on_qp;
|
||||
const std::array<int, n_outputs> out_op_dim;
|
||||
const std::array<int, n_outputs> out_offsets;
|
||||
const int output_size_on_qp;
|
||||
const int trial_vdim;
|
||||
const int total_trial_op_dim;
|
||||
const int residual_size_on_qp;
|
||||
// other constants
|
||||
const int dim, ne, nq, q1d;
|
||||
FieldDescriptor direction_fd;
|
||||
mutable Vector direction_e;
|
||||
mutable RestrictionCache<Entity::Element> direction_rcache;
|
||||
|
||||
template <std::size_t slot>
|
||||
static constexpr int ParamRank()
|
||||
{
|
||||
using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
|
||||
return qf_param_shape<param_t>::rank;
|
||||
}
|
||||
|
||||
template <std::size_t slot, int dim_idx>
|
||||
static constexpr int ParamExtent()
|
||||
{
|
||||
using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
|
||||
return qf_param_shape<param_t>::extents[dim_idx];
|
||||
}
|
||||
|
||||
template <typename fop_t, std::size_t slot>
|
||||
static constexpr int StaticVDim()
|
||||
{
|
||||
constexpr int rank = ParamRank<slot>();
|
||||
if constexpr (is_gradient_fop_v<fop_t>)
|
||||
{
|
||||
if constexpr (rank <= 1) { return 1; }
|
||||
else { return ParamExtent<slot, 0>(); }
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (rank == 0) { return 1; }
|
||||
else { return ParamExtent<slot, 0>(); }
|
||||
}
|
||||
}
|
||||
|
||||
template <typename fop_t, std::size_t slot>
|
||||
static constexpr int StaticOpDim()
|
||||
{
|
||||
constexpr int rank = ParamRank<slot>();
|
||||
if constexpr (is_gradient_fop_v<fop_t>)
|
||||
{
|
||||
if constexpr (rank == 0) { return 1; }
|
||||
else if constexpr (rank == 1) { return ParamExtent<slot, 0>(); }
|
||||
else { return ParamExtent<slot, 1>(); }
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (rank <= 1) { return 1; }
|
||||
else { return ParamExtent<slot, 1>(); }
|
||||
}
|
||||
}
|
||||
|
||||
template <std::size_t input_slot>
|
||||
static constexpr bool StaticInputDep()
|
||||
{
|
||||
using fop_t = tuple_element_t<input_slot, inputs_t>;
|
||||
return fop_t::GetFieldId() == derivative_id;
|
||||
}
|
||||
|
||||
template <std::size_t input_slot>
|
||||
static constexpr int StaticInputVDim()
|
||||
{
|
||||
using fop_t = tuple_element_t<input_slot, inputs_t>;
|
||||
return StaticVDim<fop_t, input_slot>();
|
||||
}
|
||||
|
||||
template <std::size_t input_slot>
|
||||
static constexpr int StaticInputOpDim()
|
||||
{
|
||||
using fop_t = tuple_element_t<input_slot, inputs_t>;
|
||||
return StaticOpDim<fop_t, input_slot>();
|
||||
}
|
||||
|
||||
template <std::size_t output_slot>
|
||||
static constexpr int StaticOutputVDim()
|
||||
{
|
||||
using fop_t = tuple_element_t<output_slot, outputs_t>;
|
||||
return StaticVDim<fop_t, n_inputs + output_slot>();
|
||||
}
|
||||
|
||||
template <std::size_t output_slot>
|
||||
static constexpr int StaticOutputOpDim()
|
||||
{
|
||||
using fop_t = tuple_element_t<output_slot, outputs_t>;
|
||||
return StaticOpDim<fop_t, n_inputs + output_slot>();
|
||||
}
|
||||
|
||||
template <std::size_t output_slot>
|
||||
static constexpr int StaticOutputOffset()
|
||||
{
|
||||
int offset = 0;
|
||||
for_constexpr<output_slot>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value;
|
||||
offset += StaticOutputVDim<o>() * StaticOutputOpDim<o>();
|
||||
});
|
||||
return offset;
|
||||
}
|
||||
|
||||
template <std::size_t input_slot>
|
||||
static constexpr int StaticInputOpOffset()
|
||||
{
|
||||
int offset = 0;
|
||||
for_constexpr<input_slot>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if constexpr (StaticInputDep<s>()) { offset += StaticInputOpDim<s>(); }
|
||||
});
|
||||
return offset;
|
||||
}
|
||||
|
||||
static constexpr int StaticTrialVDim()
|
||||
{
|
||||
int vdim = 1;
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if constexpr (StaticInputDep<s>()) { vdim = StaticInputVDim<s>(); }
|
||||
});
|
||||
return vdim;
|
||||
}
|
||||
|
||||
static constexpr int StaticTotalTrialOpDim()
|
||||
{
|
||||
int op_dim = 0;
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if constexpr (StaticInputDep<s>()) { op_dim += StaticInputOpDim<s>(); }
|
||||
});
|
||||
return op_dim;
|
||||
}
|
||||
|
||||
public:
|
||||
DerivativeApply() = delete;
|
||||
|
||||
DerivativeApply(IntegratorContext ctx,
|
||||
qfunc_t /*qfunc*/,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache_in):
|
||||
inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache_in),
|
||||
dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
|
||||
inputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, inputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
input_idx(create_input_vector_map(ctx, inputs)),
|
||||
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
|
||||
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
|
||||
input_vdim(get_vdim(inputs)),
|
||||
output_dtq(create_dtq_maps<Entity::Element>(
|
||||
outputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, outputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
output_idx(create_output_vector_map(ctx, outputs)),
|
||||
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
|
||||
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
|
||||
output_vdim(get_vdim(outputs)),
|
||||
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
|
||||
input_size_on_qp(
|
||||
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
|
||||
out_op_dim(compute_out_op_dim(outputs)),
|
||||
out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
|
||||
output_size_on_qp(
|
||||
[&]
|
||||
{
|
||||
int s = 0;
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{ s += get<o>(outputs).size_on_qp; });
|
||||
return s;
|
||||
}()),
|
||||
trial_vdim(compute_trial_vdim(inputs, derivative_id)),
|
||||
total_trial_op_dim(compute_total_trial_op_dim(
|
||||
inputs, input_is_dependent, input_size_on_qp)),
|
||||
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
|
||||
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
|
||||
q1d(tensor_1d_size(nq, dim))
|
||||
{
|
||||
MFEM_ASSERT(ctx.unionfds.size() == nfields,
|
||||
"LocalQFBackend: unionfds size mismatch");
|
||||
|
||||
int direction_field_idx = -1;
|
||||
for (size_t uf = 0; uf < nfields; uf++)
|
||||
{
|
||||
if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
|
||||
{
|
||||
direction_field_idx = static_cast<int>(uf);
|
||||
break;
|
||||
}
|
||||
}
|
||||
MFEM_ASSERT(direction_field_idx != -1,
|
||||
"DerivativeApply: derivative direction field not found");
|
||||
|
||||
direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename Backend>
|
||||
void run_kernels(std::vector<Vector *> &ye) const
|
||||
{
|
||||
Backend::Run(dim,
|
||||
q1d,
|
||||
ctx,
|
||||
qp_cache,
|
||||
// inputs
|
||||
input_idx,
|
||||
input_B,
|
||||
input_G,
|
||||
input_vdim,
|
||||
input_d1d,
|
||||
input_q1d,
|
||||
input_size_on_qp,
|
||||
input_is_dependent,
|
||||
// outputs
|
||||
output_idx,
|
||||
output_B,
|
||||
output_G,
|
||||
output_vdim,
|
||||
output_d1d,
|
||||
output_q1d,
|
||||
out_op_dim,
|
||||
out_offsets,
|
||||
trial_vdim,
|
||||
total_trial_op_dim,
|
||||
residual_size_on_qp,
|
||||
output_size_on_qp,
|
||||
direction_e,
|
||||
ye,
|
||||
// fallback arguments
|
||||
dim,
|
||||
q1d);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
void operator()(const std::vector<Vector *> &,
|
||||
const Vector *direction_l,
|
||||
std::vector<Vector *> &ye) const
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
MFEM_ASSERT(direction_l != nullptr,
|
||||
"LocalQF DerivativeApply: direction vector is null");
|
||||
|
||||
restriction(direction_fd,
|
||||
direction_rcache,
|
||||
*direction_l,
|
||||
direction_e,
|
||||
ElementDofOrdering::LEXICOGRAPHIC);
|
||||
if (q1d <= LocalQFLOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeApplyLO>(ye);
|
||||
}
|
||||
else if (q1d <= LocalQFHOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeApplyHO>(ye);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
|
||||
static void
|
||||
derivative_apply_callback(const IntegratorContext &ctx,
|
||||
const Vector &qp_cache,
|
||||
// inputs: idx, B, G, vdim, d1d, q1d
|
||||
const std::array<size_t, n_inputs> & /*in_idx*/,
|
||||
const std::array<const real_t *, n_inputs> in_B,
|
||||
const std::array<const real_t *, n_inputs> in_G,
|
||||
const std::array<int, n_inputs> &in_vdim,
|
||||
const std::array<int, n_inputs> &in_d1d,
|
||||
const std::array<int, n_inputs> &in_q1d,
|
||||
const std::array<int, n_inputs> &in_size_on_qp,
|
||||
const std::array<bool, n_inputs> &input_dep,
|
||||
// outputs: idx, B, G, vdim, d1d, q1d
|
||||
const std::array<size_t, n_outputs> &out_idx,
|
||||
const std::array<const real_t *, n_outputs> out_B,
|
||||
const std::array<const real_t *, n_outputs> out_G,
|
||||
const std::array<int, n_outputs> &out_vdim,
|
||||
const std::array<int, n_outputs> &out_d1d,
|
||||
const std::array<int, n_outputs> &out_q1d,
|
||||
const std::array<int, n_outputs> &out_op_dim,
|
||||
const std::array<int, n_outputs> &out_offsets,
|
||||
const int trial_vdim,
|
||||
const int total_trial_op_dim,
|
||||
const int residual_size_on_qp,
|
||||
const int output_size_on_qp,
|
||||
const Vector &direction_e,
|
||||
std::vector<Vector *> &ye,
|
||||
// fallback arguments
|
||||
const int dim,
|
||||
const int q1d)
|
||||
{
|
||||
MFEM_CONTRACT_VAR(input_dep);
|
||||
MFEM_CONTRACT_VAR(in_size_on_qp);
|
||||
MFEM_CONTRACT_VAR(out_vdim);
|
||||
MFEM_CONTRACT_VAR(out_op_dim);
|
||||
MFEM_CONTRACT_VAR(out_offsets);
|
||||
MFEM_CONTRACT_VAR(trial_vdim);
|
||||
MFEM_CONTRACT_VAR(total_trial_op_dim);
|
||||
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
static constexpr auto B2D = backend_t::DIM == 2;
|
||||
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
|
||||
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
|
||||
|
||||
const int ne = ctx.nentities;
|
||||
const int nq = ctx.ir.GetNPoints();
|
||||
MFEM_CONTRACT_VAR(output_size_on_qp);
|
||||
|
||||
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
|
||||
|
||||
// --------------------------------------------------
|
||||
// DIRECTION (trial): XE_dir for the dependent inputs
|
||||
// --------------------------------------------------
|
||||
const auto d_direction = direction_e.Read();
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if (!input_dep[i]) { return; }
|
||||
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
|
||||
{
|
||||
in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>) { /* never a direction */ }
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// --------------------------------------------------
|
||||
// OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
|
||||
// --------------------------------------------------
|
||||
std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const size_t k = out_idx[i];
|
||||
const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
|
||||
out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
|
||||
out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported FieldOperator");
|
||||
}
|
||||
});
|
||||
|
||||
auto cache_tensor = DeviceTensor<3, const real_t>(
|
||||
qp_cache.Read(), nq, residual_size_on_qp, ne);
|
||||
|
||||
const auto d_attr = ctx.attr.Read();
|
||||
const bool has_attr = ctx.attr.Size() > 0;
|
||||
const auto d_elem_attr = ctx.elem_attr->Read();
|
||||
|
||||
dfem::forall<MTPB>(
|
||||
[=] MFEM_HOST_DEVICE(const int e, void *)
|
||||
{
|
||||
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
|
||||
|
||||
// -----------------------------------------------
|
||||
// Output integration registers, trial direction (shadow) registers
|
||||
// and shared memory. `rargs` only ever holds test-function data, so
|
||||
// it is an output-only bank: slot `o` is q-function parameter
|
||||
// `n_inputs + o`.
|
||||
// -----------------------------------------------
|
||||
output_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
|
||||
input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> sargs;
|
||||
MFEM_SHARED typename backend_t::Shared smem;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Load trial direction (sargs) for the dependent inputs
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
if constexpr (!StaticInputDep<i>()) { return; }
|
||||
const auto &XE = in_XE_dir[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
|
||||
const real_t *B = in_B[i], *G = in_G[i];
|
||||
auto &sarg = get<i>(sargs);
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop<FOP>::value)
|
||||
{
|
||||
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
|
||||
using FieldParamT =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
backend_t::template LoadGradient<RNK,
|
||||
decltype(sarg),
|
||||
decltype(XE),
|
||||
FieldParamT>(
|
||||
smem, e, d, q, Q1D, B, G, XE, sarg);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP> || is_weight_fop_v<FOP>)
|
||||
{
|
||||
// identity read at qp; weight is never a trial direction
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Contract the cached Jacobian with the trial direction at each
|
||||
// quadrature point and push the result into the test registers.
|
||||
// -----------------------------------------------
|
||||
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
|
||||
// The trial direction at this quadrature point is the same
|
||||
// for every test row (i, k), so pull each dependent input
|
||||
// slot out of the register bank once, here, instead of once
|
||||
// per row inside the contraction below.
|
||||
args_tuple_t dvecs {};
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if constexpr (StaticInputDep<s>())
|
||||
{
|
||||
using SARG =
|
||||
typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
|
||||
get<s>(dvecs) = backend_t::template qp_pull<SARG>(
|
||||
get<s>(sargs), qx, qy, qz);
|
||||
}
|
||||
});
|
||||
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value, ao = n_inputs + o;
|
||||
using FOP = tuple_element_t<o, outputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
|
||||
constexpr int tv = StaticOutputVDim<o>();
|
||||
constexpr int to = StaticOutputOpDim<o>();
|
||||
constexpr int offset_o = StaticOutputOffset<o>();
|
||||
constexpr int trial_vdim_ct = StaticTrialVDim();
|
||||
constexpr int total_trial_op_dim_ct = StaticTotalTrialOpDim();
|
||||
|
||||
ARG fhat{};
|
||||
MFEM_UNROLL(tv)
|
||||
for (int i = 0; i < tv; i++)
|
||||
{
|
||||
MFEM_UNROLL(to)
|
||||
for (int k = 0; k < to; k++)
|
||||
{
|
||||
const int row = offset_o + i * to + k;
|
||||
const int cache_row =
|
||||
row * trial_vdim_ct * total_trial_op_dim_ct;
|
||||
real_t sum = 0.0;
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if constexpr (StaticInputDep<s>())
|
||||
{
|
||||
constexpr int op_dim_s = StaticInputOpDim<s>();
|
||||
constexpr int m_offset = StaticInputOpOffset<s>();
|
||||
const auto &dvec = get<s>(dvecs);
|
||||
MFEM_UNROLL(trial_vdim_ct)
|
||||
for (int j = 0; j < trial_vdim_ct; j++)
|
||||
{
|
||||
MFEM_UNROLL(op_dim_s)
|
||||
for (int m = 0; m < op_dim_s; m++)
|
||||
{
|
||||
const int cache_idx =
|
||||
cache_row + j * total_trial_op_dim_ct +
|
||||
(m + m_offset);
|
||||
sum += cache_tensor(q, cache_idx, e) *
|
||||
qf_value_at(dvec, j, m);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
qf_set_value_at(fhat, i, k, sum);
|
||||
}
|
||||
}
|
||||
|
||||
auto &YE = out_YE[o];
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
MFEM_UNROLL(tv)
|
||||
for (int i = 0; i < tv; i++)
|
||||
{
|
||||
MFEM_UNROLL(to)
|
||||
for (int k = 0; k < to; k++)
|
||||
{
|
||||
YE(i + tv * k, qx, qy, qz, e) =
|
||||
qf_value_at(fhat, i, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
backend_t::template qp_push<ARG>(
|
||||
get<o>(rargs), qx, qy, qz, fhat);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Integrate value / gradient outputs to the test dofs
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_outputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value, o = n_inputs + i;
|
||||
const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
|
||||
const auto B = out_B[i], G = out_G[i];
|
||||
auto &YE = out_YE[i];
|
||||
auto &rarg = get<i>(rargs);
|
||||
using FOP = tuple_element_t<i, outputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP>)
|
||||
{
|
||||
backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, rarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
using YE_t = decltype(YE);
|
||||
using rarg_t = decltype(rarg);
|
||||
using qf_param_t =
|
||||
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
|
||||
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
|
||||
smem, e, d, q, Q1D, B, G, YE, rarg);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>) { /* written at qp */ }
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
},
|
||||
ne,
|
||||
backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
|
||||
q1d, in_d1d, out_d1d)),
|
||||
0,
|
||||
nullptr);
|
||||
}
|
||||
|
||||
using ApplyKernelType =
|
||||
decltype(&DerivativeApply::derivative_apply_callback<>);
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyLO,
|
||||
ApplyKernelType,
|
||||
(int, int) );
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyHO,
|
||||
ApplyKernelType,
|
||||
(int, int) );
|
||||
};
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
ApplyKernelType
|
||||
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyLO::Kernel()
|
||||
{
|
||||
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
|
||||
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return apply_t::template derivative_apply_callback<
|
||||
LocalQFLOBackend<DIM, Q1D>>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
ApplyKernelType
|
||||
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyLO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeApplyLO = typename apply_t::DerivativeApplyLO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeApplyLO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeApplyLO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
ApplyKernelType
|
||||
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyHO::Kernel()
|
||||
{
|
||||
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return apply_t::template derivative_apply_callback<LocalQFHOBackend<DIM>,
|
||||
Q1D>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
ApplyKernelType
|
||||
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyHO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeApplyHO = typename apply_t::DerivativeApplyHO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeApplyHO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeApplyHO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future::LocalQFImpl
|
||||
@@ -1,637 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
|
||||
#include "kernels.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
#include <array>
|
||||
|
||||
namespace mfem::future::LocalQFImpl
|
||||
{
|
||||
|
||||
// Cached transposed Jacobian apply: Jᵀ·w from the qp_cache
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
class DerivativeApplyTranspose
|
||||
{
|
||||
static constexpr auto inout_tuple =
|
||||
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
|
||||
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
|
||||
static constexpr size_t nfields =
|
||||
count_unique_field_ids(filtered_inout_tuple);
|
||||
|
||||
using qf_signature = typename get_function_signature<qfunc_t>::type;
|
||||
using qf_param_ts = typename qf_signature::parameter_ts;
|
||||
using args_tuple_t = decay_tuple<qf_param_ts>;
|
||||
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
|
||||
"LocalQF: q-function arity must match inputs + outputs");
|
||||
|
||||
// Input tuple slot referencing the derivative field (compile-time)
|
||||
static constexpr size_t deriv_input_idx_ct = []() constexpr
|
||||
{
|
||||
size_t idx = SIZE_MAX;
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if (FOP::GetFieldId() == derivative_id) { idx = i; }
|
||||
});
|
||||
return idx;
|
||||
}();
|
||||
static_assert(deriv_input_idx_ct < n_inputs,
|
||||
"DerivativeApplyTranspose: derivative input slot not found");
|
||||
|
||||
const inputs_t inputs;
|
||||
const outputs_t outputs;
|
||||
const IntegratorContext ctx;
|
||||
const Vector &qp_cache; // Jacobian cache from DerivativeSetup
|
||||
const std::vector<const DofToQuad *> dtqs;
|
||||
// inputs: dtq, B, G, d1d, q1d, vdim (trial / derivative fields)
|
||||
const std::array<DofToQuadMap, n_inputs> input_dtq;
|
||||
const std::array<const real_t *, n_inputs> input_B, input_G;
|
||||
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
|
||||
// outputs: dtq, idx, B, G, d1d, q1d, vdim (test / cotangent fields)
|
||||
const std::array<DofToQuadMap, n_outputs> output_dtq;
|
||||
const std::array<size_t, n_outputs> output_idx;
|
||||
const std::array<const real_t *, n_outputs> output_B, output_G;
|
||||
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
|
||||
// Jacobian cache metadata
|
||||
const std::array<bool, n_inputs> input_is_dependent;
|
||||
const std::array<int, n_inputs> input_size_on_qp;
|
||||
const std::array<int, n_outputs> out_op_dim;
|
||||
const std::array<int, n_outputs> out_offsets;
|
||||
const int output_size_on_qp;
|
||||
const int trial_vdim;
|
||||
const int total_trial_op_dim;
|
||||
const int residual_size_on_qp;
|
||||
// other constants
|
||||
const int dim, ne, nq, q1d;
|
||||
const size_t deriv_infd_idx; // index of the derivative field in ye
|
||||
// output cotangent restriction workspace (blocked by element)
|
||||
std::array<int, n_outputs> out_elem_dof_size;
|
||||
mutable Vector dir_out_e;
|
||||
/// One restriction cache per output field, resolved on first use.
|
||||
mutable std::array<RestrictionCache<Entity::Element>, n_outputs>
|
||||
out_rcaches;
|
||||
|
||||
public:
|
||||
//////////////////////////////////////////////////////////////////
|
||||
DerivativeApplyTranspose() = delete;
|
||||
|
||||
DerivativeApplyTranspose(IntegratorContext ctx,
|
||||
qfunc_t /*qfunc*/,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache):
|
||||
inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache),
|
||||
dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
|
||||
inputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, inputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
|
||||
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
|
||||
input_vdim(get_vdim(inputs)),
|
||||
output_dtq(create_dtq_maps<Entity::Element>(
|
||||
outputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, outputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
output_idx(create_output_vector_map(ctx, outputs)),
|
||||
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
|
||||
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
|
||||
output_vdim(get_vdim(outputs)),
|
||||
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
|
||||
input_size_on_qp(
|
||||
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
|
||||
out_op_dim(compute_out_op_dim(outputs)),
|
||||
out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
|
||||
output_size_on_qp(
|
||||
[&]
|
||||
{
|
||||
int s = 0;
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{ s += get<o>(outputs).size_on_qp; });
|
||||
return s;
|
||||
}()),
|
||||
trial_vdim(compute_trial_vdim(inputs, derivative_id)),
|
||||
total_trial_op_dim(compute_total_trial_op_dim(
|
||||
inputs, input_is_dependent, input_size_on_qp)),
|
||||
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
|
||||
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
|
||||
q1d(tensor_1d_size(nq, dim)),
|
||||
deriv_infd_idx(find_infd_index(ctx, derivative_id)), out_elem_dof_size{}
|
||||
{
|
||||
MFEM_ASSERT(ctx.unionfds.size() == nfields,
|
||||
"LocalQFBackend: unionfds size mismatch");
|
||||
MFEM_ASSERT(
|
||||
deriv_infd_idx != SIZE_MAX,
|
||||
"DerivativeApplyTranspose: derivative field not found in infds");
|
||||
|
||||
// Size the workspace that holds the output cotangent(s) in element
|
||||
// layout.
|
||||
int total_dir_e_size = 0;
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{
|
||||
const int elem_sz = compute_element_dof_sz(
|
||||
ctx.outfds[output_idx[o]], ne, ElementDofOrdering::LEXICOGRAPHIC);
|
||||
out_elem_dof_size[o] = elem_sz;
|
||||
total_dir_e_size += elem_sz;
|
||||
});
|
||||
dir_out_e.SetSize(total_dir_e_size * ne);
|
||||
dir_out_e.UseDevice(true);
|
||||
dir_out_e.Read();
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename Backend>
|
||||
void run_kernels(std::vector<Vector *> &ye) const
|
||||
{
|
||||
Backend::Run(dim,
|
||||
q1d,
|
||||
ctx,
|
||||
qp_cache,
|
||||
dir_out_e,
|
||||
// inputs (integration target metadata)
|
||||
input_B,
|
||||
input_G,
|
||||
input_vdim,
|
||||
input_d1d,
|
||||
input_q1d,
|
||||
input_size_on_qp,
|
||||
input_is_dependent,
|
||||
// outputs (direction interpolation metadata)
|
||||
output_B,
|
||||
output_G,
|
||||
output_vdim,
|
||||
output_d1d,
|
||||
output_q1d,
|
||||
out_op_dim,
|
||||
out_offsets,
|
||||
trial_vdim,
|
||||
total_trial_op_dim,
|
||||
residual_size_on_qp,
|
||||
output_size_on_qp,
|
||||
deriv_infd_idx,
|
||||
ye,
|
||||
// fallback arguments
|
||||
dim,
|
||||
q1d);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
void operator()(const std::vector<Vector *> & /*xe*/,
|
||||
const Vector *direction_l,
|
||||
std::vector<Vector *> &ye) const
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
MFEM_ASSERT(direction_l != nullptr,
|
||||
"LocalQF DerivativeApplyTranspose: direction vector is null");
|
||||
|
||||
// Restrict output cotangent from L-vectors into element layout
|
||||
// (dir_out_e).
|
||||
int l_offset = 0;
|
||||
int e_offset = 0;
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{
|
||||
const size_t outfd = output_idx[o];
|
||||
const auto &fd = ctx.outfds[outfd];
|
||||
const int l_size = GetVSize(fd);
|
||||
Vector dir_o_l(*const_cast<Vector *>(direction_l), l_offset, l_size);
|
||||
dir_o_l.UseDevice(true);
|
||||
const int elem_sz = out_elem_dof_size[o];
|
||||
Vector dir_o_e(dir_out_e, e_offset, elem_sz * ne);
|
||||
dir_o_e.UseDevice(true);
|
||||
restriction(fd, out_rcaches[o], dir_o_l, dir_o_e,
|
||||
ElementDofOrdering::LEXICOGRAPHIC);
|
||||
l_offset += l_size;
|
||||
e_offset += elem_sz * ne;
|
||||
});
|
||||
|
||||
if (q1d <= LocalQFLOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeApplyTransposeLO>(ye);
|
||||
}
|
||||
else if (q1d <= LocalQFHOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeApplyTransposeHO>(ye);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
|
||||
static void derivative_apply_transpose_callback(
|
||||
const IntegratorContext &ctx,
|
||||
const Vector &qp_cache,
|
||||
const Vector &dir_e, // restricted, concatenated output cotangents
|
||||
// inputs (integration target metadata)
|
||||
const std::array<const real_t *, n_inputs> in_B,
|
||||
const std::array<const real_t *, n_inputs> in_G,
|
||||
const std::array<int, n_inputs> &in_vdim,
|
||||
const std::array<int, n_inputs> &in_d1d,
|
||||
const std::array<int, n_inputs> &in_q1d,
|
||||
const std::array<int, n_inputs> &in_size_on_qp,
|
||||
const std::array<bool, n_inputs> &input_dep,
|
||||
// outputs (direction interpolation metadata)
|
||||
const std::array<const real_t *, n_outputs> out_B,
|
||||
const std::array<const real_t *, n_outputs> out_G,
|
||||
const std::array<int, n_outputs> &out_vdim,
|
||||
const std::array<int, n_outputs> &out_d1d,
|
||||
const std::array<int, n_outputs> &out_q1d,
|
||||
const std::array<int, n_outputs> &out_op_dim,
|
||||
const std::array<int, n_outputs> &out_offsets,
|
||||
const int trial_vdim,
|
||||
const int total_trial_op_dim,
|
||||
const int residual_size_on_qp,
|
||||
const int output_size_on_qp,
|
||||
const size_t deriv_infd_idx,
|
||||
std::vector<Vector *> &ye,
|
||||
// fallback arguments
|
||||
const int dim,
|
||||
const int q1d)
|
||||
{
|
||||
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
static constexpr auto B2D = backend_t::DIM == 2;
|
||||
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
|
||||
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
|
||||
|
||||
const int ne = ctx.nentities;
|
||||
const int nq = ctx.ir.GetNPoints();
|
||||
MFEM_CONTRACT_VAR(output_size_on_qp);
|
||||
MFEM_CONTRACT_VAR(in_q1d);
|
||||
|
||||
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
|
||||
|
||||
// --------------------------------------------------
|
||||
// DIRECTION (test cotangent): out_XE_dir, concatenated per output
|
||||
// --------------------------------------------------
|
||||
const auto d_dir = dir_e.Read();
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_outputs> out_XE_dir;
|
||||
int e_offset = 0;
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value;
|
||||
const int d = out_d1d[o], q = out_q1d[o], v = out_vdim[o];
|
||||
using FOP = tuple_element_t<o, outputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
|
||||
{
|
||||
out_XE_dir[o] = Reshape(d_dir + e_offset, d, d, B2D ? 1 : d, v, ne);
|
||||
e_offset += k_dim(d) * v * ne;
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
out_XE_dir[o] = Reshape(d_dir + e_offset, v, q, q, B2D ? 1 : q, ne);
|
||||
e_offset += k_dim(q) * v * ne;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
// --------------------------------------------------
|
||||
// DERIVATIVE TRIAL FIELD: ye_XE (accumulates Jᵀ w)
|
||||
// --------------------------------------------------
|
||||
const int d_in = in_d1d[deriv_input_idx_ct];
|
||||
const int v_in = in_vdim[deriv_input_idx_ct];
|
||||
auto ye_XE = Reshape(
|
||||
ye[deriv_infd_idx]->ReadWrite(), d_in, d_in, B2D ? 1 : d_in, v_in, ne);
|
||||
|
||||
auto cache_tensor = DeviceTensor<3, const real_t>(
|
||||
qp_cache.Read(), nq, residual_size_on_qp, ne);
|
||||
|
||||
const auto d_attr = ctx.attr.Read();
|
||||
const bool has_attr = ctx.attr.Size() > 0;
|
||||
const auto d_elem_attr = ctx.elem_attr->Read();
|
||||
|
||||
dfem::forall<MTPB>(
|
||||
[=] MFEM_HOST_DEVICE(const int e, void *)
|
||||
{
|
||||
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
|
||||
|
||||
// -----------------------------------------------
|
||||
// Output cotangent (direction) registers live in the output slots;
|
||||
// the trial integration data is pushed into the input slots.
|
||||
// -----------------------------------------------
|
||||
args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
|
||||
MFEM_SHARED typename backend_t::Shared smem;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Interpolate the test cotangent to quadrature points (output slots)
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value, ao = n_inputs + o;
|
||||
using FOP = tuple_element_t<o, outputs_t>;
|
||||
const auto &XE = out_XE_dir[o];
|
||||
const int d = out_d1d[o], q = out_q1d[o], Q1D = q1d;
|
||||
const real_t *B = out_B[o], *G = out_G[o];
|
||||
auto &oarg = get<ao>(rargs);
|
||||
if constexpr (is_value_fop_v<FOP>)
|
||||
{
|
||||
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, oarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, ao>::extents.size();
|
||||
using FieldParamT =
|
||||
typename qf_param_slot<qfunc_t, ao>::qf_decay_param_t;
|
||||
backend_t::template LoadGradient<RNK,
|
||||
decltype(oarg),
|
||||
decltype(XE),
|
||||
FieldParamT>(
|
||||
smem, e, d, q, Q1D, B, G, XE, oarg);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
// identity cotangent is read directly at qp from out_XE_dir
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Contract the transposed cached Jacobian with the test cotangent at
|
||||
// each quadrature point and push the trial result into the dependent
|
||||
// input registers.
|
||||
// -----------------------------------------------
|
||||
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
|
||||
// The test cotangent at this quadrature point is the same
|
||||
// for every trial column (j, m), so pull each interpolated
|
||||
// output slot out of the register bank once, here, instead
|
||||
// of once per column inside the contraction below. Identity
|
||||
// outputs have no register bank and are read from
|
||||
// out_XE_dir at the point of use.
|
||||
args_tuple_t wvecs {};
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value, ao = n_inputs + o;
|
||||
using OFOP = tuple_element_t<o, outputs_t>;
|
||||
if constexpr (is_value_fop_v<OFOP> ||
|
||||
is_gradient_fop_v<OFOP>)
|
||||
{
|
||||
using OARG =
|
||||
typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
|
||||
get<ao>(wvecs) = backend_t::template qp_pull<OARG>(
|
||||
get<ao>(rargs), qx, qy, qz);
|
||||
}
|
||||
});
|
||||
|
||||
int m_offset = 0;
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if (!input_dep[s]) { return; }
|
||||
using SARG =
|
||||
typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
|
||||
const int vdim_s = in_vdim[s];
|
||||
const int op_dim_s = in_size_on_qp[s] / vdim_s;
|
||||
|
||||
SARG fhat{};
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
for (int m = 0; m < op_dim_s; m++)
|
||||
{
|
||||
const int col =
|
||||
j * total_trial_op_dim + (m + m_offset);
|
||||
real_t sum = 0.0;
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value, ao = n_inputs + o;
|
||||
using OFOP = tuple_element_t<o, outputs_t>;
|
||||
const int tv = out_vdim[o], to = out_op_dim[o];
|
||||
const auto offset_o = out_offsets[o];
|
||||
const auto &cache = cache_tensor;
|
||||
if constexpr (is_value_fop_v<OFOP> ||
|
||||
is_gradient_fop_v<OFOP>)
|
||||
{
|
||||
const auto &wvec = get<ao>(wvecs);
|
||||
for (int i = 0; i < tv; i++)
|
||||
{
|
||||
for (int k = 0; k < to; k++)
|
||||
{
|
||||
const int row = offset_o + i * to + k;
|
||||
const int cache_idx =
|
||||
row * trial_vdim *
|
||||
total_trial_op_dim +
|
||||
col;
|
||||
sum += cache(q, cache_idx, e) *
|
||||
qf_value_at(wvec, i, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<OFOP>)
|
||||
{
|
||||
const auto &XEo = out_XE_dir[o];
|
||||
for (int i = 0; i < tv; i++)
|
||||
{
|
||||
for (int k = 0; k < to; k++)
|
||||
{
|
||||
const int row = offset_o + i * to + k;
|
||||
const int cache_idx =
|
||||
row * trial_vdim *
|
||||
total_trial_op_dim +
|
||||
col;
|
||||
sum += cache(q, cache_idx, e) *
|
||||
XEo(i + tv * k, qx, qy, qz, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
qf_set_value_at(fhat, j, m, sum);
|
||||
}
|
||||
}
|
||||
backend_t::template qp_push<SARG>(
|
||||
get<s>(rargs), qx, qy, qz, fhat);
|
||||
m_offset += op_dim_s;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Integrate the trial result into the derivative field dofs. Multiple
|
||||
// dependent input slots (e.g. value and gradient of the same field)
|
||||
// accumulate into ye_XE via the writers' '+=' semantics.
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if (!input_dep[s]) { return; }
|
||||
using FOP = tuple_element_t<s, inputs_t>;
|
||||
const int d = in_d1d[s], q = in_q1d[s], Q1D = q1d;
|
||||
const real_t *B = in_B[s], *G = in_G[s];
|
||||
auto &sarg = get<s>(rargs);
|
||||
auto &YE = ye_XE;
|
||||
if constexpr (is_value_fop_v<FOP>)
|
||||
{
|
||||
backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, sarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
using YE_t = decltype(YE);
|
||||
using rarg_t = decltype(sarg);
|
||||
using qf_param_t =
|
||||
typename qf_param_slot<qfunc_t, s>::qf_decay_param_t;
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, s>::extents.size();
|
||||
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
|
||||
smem, e, d, q, Q1D, B, G, YE, sarg);
|
||||
}
|
||||
else
|
||||
{
|
||||
// identity / weight derivative targets are not produced here
|
||||
}
|
||||
});
|
||||
},
|
||||
ne,
|
||||
backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
|
||||
q1d, in_d1d, out_d1d)),
|
||||
0,
|
||||
nullptr);
|
||||
}
|
||||
|
||||
using TransposeKernelType =
|
||||
decltype(&DerivativeApplyTranspose::
|
||||
derivative_apply_transpose_callback<>);
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeLO,
|
||||
TransposeKernelType,
|
||||
(int, int) );
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeHO,
|
||||
TransposeKernelType,
|
||||
(int, int) );
|
||||
};
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeApplyTranspose<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::TransposeKernelType
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyTransposeLO::Kernel()
|
||||
{
|
||||
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
|
||||
using transpose_t =
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return transpose_t::template derivative_apply_transpose_callback<
|
||||
LocalQFLOBackend<DIM, Q1D>>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeApplyTranspose<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::TransposeKernelType
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyTransposeLO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using transpose_t =
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeApplyTransposeLO =
|
||||
typename transpose_t::DerivativeApplyTransposeLO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeApplyTranspose<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::TransposeKernelType
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyTransposeHO::Kernel()
|
||||
{
|
||||
using transpose_t =
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return transpose_t::
|
||||
template derivative_apply_transpose_callback<LocalQFHOBackend<DIM>, Q1D>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeApplyTranspose<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::TransposeKernelType
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeApplyTransposeHO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using transpose_t =
|
||||
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeApplyTransposeHO =
|
||||
typename transpose_t::DerivativeApplyTransposeHO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future::LocalQFImpl
|
||||
@@ -1,987 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
#include "../../../kernels.hpp"
|
||||
|
||||
#include "kernels.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <type_traits>
|
||||
|
||||
namespace ker = mfem::kernels::internal;
|
||||
|
||||
namespace mfem::future::LocalQFImpl
|
||||
{
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
template<int DIM>
|
||||
MFEM_HOST_DEVICE inline int tensor_idx(int x, int y, int z, int N)
|
||||
{
|
||||
static_assert(DIM == 2 || DIM == 3);
|
||||
if constexpr (DIM == 2) { assert(z == 0); }
|
||||
return x + N * (y + N * z);
|
||||
}
|
||||
|
||||
template<int DIM>
|
||||
MFEM_HOST_DEVICE inline real_t
|
||||
trial_basis_weight_value(const DeviceTensor<3, const real_t> &B,
|
||||
const int qx,
|
||||
const int qy,
|
||||
const int qz,
|
||||
const int Jx,
|
||||
const int Jy,
|
||||
const int Jz)
|
||||
{
|
||||
static_assert(DIM == 2 || DIM == 3);
|
||||
return B(qx, 0, Jx) * B(qy, 0, Jy) * ((DIM == 3) ? B(qz, 0, Jz) : 1.0);
|
||||
}
|
||||
|
||||
template<int DIM>
|
||||
MFEM_HOST_DEVICE inline real_t
|
||||
trial_basis_weight_gradient(const DeviceTensor<3, const real_t> &B,
|
||||
const DeviceTensor<3, const real_t> &G,
|
||||
const int m,
|
||||
const int qx,
|
||||
const int qy,
|
||||
const int qz,
|
||||
const int Jx,
|
||||
const int Jy,
|
||||
const int Jz)
|
||||
{
|
||||
const auto Gx = G(qx, 0, Jx), Gy = G(qy, 0, Jy);
|
||||
const auto Bx = B(qx, 0, Jx), By = B(qy, 0, Jy);
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
MFEM_CONTRACT_VAR(qz & Jz);
|
||||
return (m == 0) ? Gx * By : Bx * Gy;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto Bz = B(qz, 0, Jz), Gz = G(qz, 0, Jz);
|
||||
return (m == 0) ? Gx * By * Bz
|
||||
: (m == 1) ? Bx * Gy * Bz
|
||||
: (m == 2) ? Bx * By * Gz
|
||||
: (assert(false), 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
template<int DIM, int MQ1, typename Shared, typename output_t>
|
||||
MFEM_HOST_DEVICE void
|
||||
map_quadrature_data_to_fields(DeviceTensor<2, real_t> &y,
|
||||
const DeviceTensor<3, real_t> &f,
|
||||
const output_t &output,
|
||||
const DofToQuadMap &dtq,
|
||||
Shared &s,
|
||||
const int tv_dof = -1)
|
||||
{
|
||||
using output_fop_t = std::decay_t<output_t>;
|
||||
const auto B = dtq.B, G = dtq.G;
|
||||
const bool f_slab = (tv_dof >= 0);
|
||||
const int vdim = output.vdim;
|
||||
const int vd_begin = f_slab ? tv_dof : 0;
|
||||
const int vd_end = f_slab ? tv_dof + 1 : vdim;
|
||||
|
||||
if constexpr (is_value_fop_v<output_fop_t>)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
MFEM_CONTRACT_VAR(unused);
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
MFEM_CONTRACT_VAR(test_dim);
|
||||
const int f_vdim = f_slab ? 1 : vdim;
|
||||
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
|
||||
ker::LoadMatrix(d1d, q1d, B, s.B);
|
||||
ker::s_regs2d_t<MQ1> r_qp, Y;
|
||||
for (int vd = vd_begin; vd < vd_end; vd++)
|
||||
{
|
||||
const int fi = f_slab ? 0 : vd;
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{ r_qp[qy][qx] = fqp(fi, 0, qx, qy); }
|
||||
MFEM_SYNC_THREAD;
|
||||
ker::Eval2d<MQ1, true>(d1d, q1d, s.M, s.B, r_qp, Y);
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d) { yd(dx, dy, vd) += Y[dy][dx]; }
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
|
||||
ker::LoadMatrix(d1d, q1d, B, s.B);
|
||||
ker::s_regs3d_t<MQ1> f_qp, Y;
|
||||
for (int vd = vd_begin; vd < vd_end; vd++)
|
||||
{
|
||||
const int fi = f_slab ? 0 : vd;
|
||||
for (int qz = 0; qz < q1d; qz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{ f_qp[qz][qy][qx] = fqp(fi, 0, qx, qy, qz); }
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
ker::Eval3d<MQ1, true>(d1d, q1d, s.M, s.B, f_qp, Y);
|
||||
for (int dz = 0; dz < d1d; dz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{ yd(dx, dy, dz, vd) += Y[dz][dy][dx]; }
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<output_fop_t>)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = G.GetShape();
|
||||
MFEM_CONTRACT_VAR(unused);
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
const int f_vdim = f_slab ? 1 : vdim;
|
||||
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
|
||||
ker::LoadMatrix(d1d, q1d, B, s.B);
|
||||
ker::LoadMatrix(d1d, q1d, G, s.G);
|
||||
ker::vd_regs2d_t<1, DIM, MQ1> X, Y;
|
||||
for (int vd = vd_begin; vd < vd_end; vd++)
|
||||
{
|
||||
const int fi = f_slab ? 0 : vd;
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
for (int k = 0; k < DIM; k++)
|
||||
{
|
||||
X[0][k][qy][qx] = fqp(fi, k, qx, qy);
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
ker::Grad2d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t u = 0.0;
|
||||
for (int k = 0; k < DIM; k++) { u += Y[0][k][dy][dx]; }
|
||||
yd(dx, dy, vd) += u;
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
|
||||
ker::LoadMatrix(d1d, q1d, B, s.B);
|
||||
ker::LoadMatrix(d1d, q1d, G, s.G);
|
||||
ker::vd_regs3d_t<1, DIM, MQ1> X, Y;
|
||||
for (int vd = vd_begin; vd < vd_end; vd++)
|
||||
{
|
||||
const int fi = f_slab ? 0 : vd;
|
||||
for (int qz = 0; qz < q1d; qz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
for (int k = 0; k < DIM; k++)
|
||||
{
|
||||
X[0][k][qz][qy][qx] = fqp(fi, k, qx, qy, qz);
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
ker::Grad3d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
|
||||
for (int dz = 0; dz < d1d; dz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t u = 0.0;
|
||||
for (int k = 0; k < DIM; k++) { u += Y[0][k][dz][dy][dx]; }
|
||||
yd(dx, dy, dz, vd) += u;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<output_fop_t>)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
MFEM_CONTRACT_VAR(unused);
|
||||
MFEM_CONTRACT_VAR(d1d);
|
||||
|
||||
const int f_sq = f_slab ? 1 : output.size_on_qp;
|
||||
const int sq_begin = f_slab ? tv_dof : 0;
|
||||
const int sq_end = f_slab ? tv_dof + 1 : output.size_on_qp;
|
||||
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d);
|
||||
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
|
||||
for (int sq = sq_begin; sq < sq_end; sq++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
int qz = 0;
|
||||
MFEM_CONTRACT_VAR(qz);
|
||||
yqp(sq, qx, qy) = fqp(0, qx, qy);
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d, q1d);
|
||||
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
|
||||
for (int sq = sq_begin; sq < sq_end; sq++)
|
||||
{
|
||||
for (int qz = 0; qz < q1d; qz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{ yqp(sq, qx, qy, qz) = fqp(0, qx, qy, qz); }
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
|
||||
" for this field descriptor with sum factorization on"
|
||||
" tensor product elements");
|
||||
}
|
||||
}
|
||||
|
||||
template<int DIM,
|
||||
int MQ1,
|
||||
typename Shared,
|
||||
typename input_fop_ts,
|
||||
std::size_t n_inputs,
|
||||
typename output_fop_t>
|
||||
MFEM_HOST_DEVICE void assemble_element_mat_sumfact(
|
||||
const DeviceTensor<5, real_t> &Ae,
|
||||
const DeviceTensor<5, const real_t> &qpdc,
|
||||
const int e,
|
||||
const DeviceTensor<1, const real_t> &itod,
|
||||
const input_fop_ts &inputs,
|
||||
const output_fop_t &output,
|
||||
const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
|
||||
const DofToQuadMap &output_dtq,
|
||||
const int row_offset,
|
||||
const int test_vdim,
|
||||
const int test_op_dim,
|
||||
const int q1d,
|
||||
const int num_trial_dof_1d,
|
||||
real_t *fhat_storage,
|
||||
Shared &smem)
|
||||
{
|
||||
static constexpr int MQN = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
|
||||
// Slab must hold full (test_vdim, test_op_dim, nq) fhat.
|
||||
// It is allocated by the caller, and is shared by every output, so it must be
|
||||
// Before, declaring it here allocated one slab per output and the device
|
||||
// kernel ran out of shared memory once an integrator had more outputs.
|
||||
static constexpr int FHAT_SLAB_MAX = MQN * 4;
|
||||
|
||||
static constexpr bool grad_out = is_gradient_fop_v<output_fop_t>;
|
||||
static constexpr bool ident_out = is_identity_fop_v<output_fop_t>;
|
||||
|
||||
// qpdc shape: (nq, total_trial_op_dim, trial_vdim, output_size_on_qp, ne),
|
||||
// where output_size_on_qp spans every output FieldOperator (multi-output mode).
|
||||
// The rows of one output start at @a row_offset and are laid out as
|
||||
// i * test_op_dim + k, matching how DerivativeSetup writes the cache.
|
||||
const int trial_vdim = qpdc.GetShape()[2];
|
||||
const int num_test_dof = Ae.GetShape()[0];
|
||||
const int nq = qpdc.GetShape()[0];
|
||||
const int size_on_qp = output.size_on_qp;
|
||||
|
||||
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
|
||||
MFEM_VERIFY(test_op_dim <= DIM,
|
||||
"DerivativeAssemble: test_op_dim exceeds spatial DIM");
|
||||
MFEM_VERIFY(test_op_dim * nq <= FHAT_SLAB_MAX,
|
||||
"DerivativeAssemble: fhat slab exceeds capacity");
|
||||
#endif
|
||||
|
||||
const auto &inputs_ref = inputs;
|
||||
|
||||
// Iterate quadrature points using the thread-block mapping
|
||||
const auto foreach_qp = [&](auto &&body)
|
||||
{
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d) { body(qx, qy, 0); }
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int qz = 0; qz < q1d; qz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d) { body(qx, qy, qz); }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const auto zero_slab = [&](const int n_comp)
|
||||
{
|
||||
foreach_qp([&](const int qx, const int qy, const int qz)
|
||||
{
|
||||
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
|
||||
for (int k = 0; k < n_comp; k++) { fhat_storage[k * nq + q] = 0.0; }
|
||||
});
|
||||
MFEM_SYNC_THREAD;
|
||||
};
|
||||
|
||||
const auto accumulate_tv = [&](const int Jx,
|
||||
const int Jy,
|
||||
const int Jz,
|
||||
const int j,
|
||||
const int tv,
|
||||
const int tod_only = -1)
|
||||
{
|
||||
int m_offset = 0;
|
||||
for_constexpr<n_inputs>([&](auto inp)
|
||||
{
|
||||
using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
|
||||
|
||||
const int trial_op_dim = static_cast<int>(itod(static_cast<int>(inp)));
|
||||
if (trial_op_dim == 0) { return; }
|
||||
|
||||
const auto &B = input_dtq_maps[inp].B;
|
||||
const auto &G = input_dtq_maps[inp].G;
|
||||
|
||||
if constexpr (is_value_fop<fop_t>::value)
|
||||
{
|
||||
foreach_qp([&](const int qx, const int qy, const int qz)
|
||||
{
|
||||
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
|
||||
const real_t w =
|
||||
trial_basis_weight_value<DIM>(B, qx, qy, qz, Jx, Jy, Jz);
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
if (tod_only >= 0 && k != tod_only) { continue; }
|
||||
const real_t f = qpdc(q, m + m_offset, j, row_offset + tv * test_op_dim + k, e);
|
||||
if constexpr (grad_out && !ident_out)
|
||||
{
|
||||
fhat_storage[k * nq + q] += f * w;
|
||||
}
|
||||
else
|
||||
{
|
||||
fhat_storage[q] += f * w;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
else if constexpr (is_gradient_fop<fop_t>::value)
|
||||
{
|
||||
foreach_qp([&](const int qx, const int qy, const int qz)
|
||||
{
|
||||
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
const real_t w = trial_basis_weight_gradient<DIM>(
|
||||
B, G, m, qx, qy, qz, Jx, Jy, Jz);
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
if (tod_only >= 0 && k != tod_only) { continue; }
|
||||
const real_t f = qpdc(q, m + m_offset, j, row_offset + tv * test_op_dim + k, e);
|
||||
if constexpr (grad_out && !ident_out)
|
||||
{
|
||||
fhat_storage[k * nq + q] += f * w;
|
||||
}
|
||||
else
|
||||
{
|
||||
fhat_storage[q] += f * w;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("sum factorized sparse matrix assemble routine "
|
||||
"not implemented for field operator");
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
m_offset += trial_op_dim;
|
||||
});
|
||||
};
|
||||
|
||||
for (int Jz = 0; Jz < ((DIM == 2) ? 1 : num_trial_dof_1d); Jz++)
|
||||
{
|
||||
for (int Jy = 0; Jy < num_trial_dof_1d; Jy++)
|
||||
{
|
||||
for (int Jx = 0; Jx < num_trial_dof_1d; Jx++)
|
||||
{
|
||||
const int J = tensor_idx<DIM>(Jx, Jy, Jz, num_trial_dof_1d);
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
auto bvtfhat =
|
||||
Reshape(&Ae(0, 0, J, j, e), num_test_dof, test_vdim);
|
||||
const int fhat_size = test_vdim * test_op_dim * nq;
|
||||
|
||||
if (fhat_size <= FHAT_SLAB_MAX)
|
||||
{
|
||||
auto fhat =
|
||||
Reshape(&fhat_storage[0], test_vdim, test_op_dim, nq);
|
||||
for (int tv = 0; tv < test_vdim; tv++)
|
||||
{
|
||||
for (int tod = 0; tod < test_op_dim; tod++)
|
||||
{
|
||||
foreach_qp([&](const int qx, const int qy, const int qz)
|
||||
{
|
||||
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
|
||||
fhat(tv, tod, q) = 0.0;
|
||||
});
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
int m_offset = 0;
|
||||
for_constexpr<n_inputs>([&](auto inp)
|
||||
{
|
||||
using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
|
||||
|
||||
const int trial_op_dim =
|
||||
static_cast<int>(itod(static_cast<int>(inp)));
|
||||
if (trial_op_dim == 0) { return; }
|
||||
|
||||
const auto &B = input_dtq_maps[inp].B;
|
||||
const auto &G = input_dtq_maps[inp].G;
|
||||
|
||||
if constexpr (is_value_fop<fop_t>::value)
|
||||
{
|
||||
foreach_qp([&](const int qx, const int qy, const int qz)
|
||||
{
|
||||
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
|
||||
const real_t w = trial_basis_weight_value<DIM>(
|
||||
B, qx, qy, qz, Jx, Jy, Jz);
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
const real_t f =
|
||||
qpdc(q, m + m_offset, j, row_offset + i * test_op_dim + k, e);
|
||||
fhat(i, k, q) += f * w;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
else if constexpr (is_gradient_fop<fop_t>::value)
|
||||
{
|
||||
foreach_qp([&](const int qx, const int qy, const int qz)
|
||||
{
|
||||
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
const real_t w = trial_basis_weight_gradient<DIM>(
|
||||
B, G, m, qx, qy, qz, Jx, Jy, Jz);
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
const real_t f =
|
||||
qpdc(q, m + m_offset, j, row_offset + i * test_op_dim + k, e);
|
||||
fhat(i, k, q) += f * w;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL(
|
||||
"sum factorized sparse matrix assemble routine "
|
||||
"not implemented for field operator");
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
m_offset += trial_op_dim;
|
||||
});
|
||||
map_quadrature_data_to_fields<DIM, MQ1>(
|
||||
bvtfhat, fhat, output, output_dtq, smem);
|
||||
}
|
||||
else if constexpr (ident_out)
|
||||
{
|
||||
for (int sq = 0; sq < size_on_qp; sq++)
|
||||
{
|
||||
const int tv = sq / test_op_dim;
|
||||
const int tod = sq % test_op_dim;
|
||||
zero_slab(1);
|
||||
accumulate_tv(Jx, Jy, Jz, j, tv, tod);
|
||||
auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
|
||||
map_quadrature_data_to_fields<DIM, MQ1>(
|
||||
bvtfhat, f_slab, output, output_dtq, smem, sq);
|
||||
}
|
||||
}
|
||||
else if constexpr (grad_out)
|
||||
{
|
||||
for (int tv = 0; tv < test_vdim; tv++)
|
||||
{
|
||||
zero_slab(test_op_dim);
|
||||
accumulate_tv(Jx, Jy, Jz, j, tv);
|
||||
auto f_slab =
|
||||
Reshape(&fhat_storage[0], 1, test_op_dim, nq);
|
||||
map_quadrature_data_to_fields<DIM, MQ1>(
|
||||
bvtfhat, f_slab, output, output_dtq, smem, tv);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int tv = 0; tv < test_vdim; tv++)
|
||||
{
|
||||
zero_slab(1);
|
||||
accumulate_tv(Jx, Jy, Jz, j, tv);
|
||||
auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
|
||||
map_quadrature_data_to_fields<DIM, MQ1>(
|
||||
bvtfhat, f_slab, output, output_dtq, smem, tv);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Assemble sparse Jacobian from cached quadrature derivatives (tensor 2D/3D)
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
class DerivativeAssemble
|
||||
{
|
||||
static constexpr auto inout_tuple =
|
||||
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
|
||||
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
|
||||
static constexpr size_t nfields =
|
||||
count_unique_field_ids(filtered_inout_tuple);
|
||||
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
|
||||
const IntegratorContext ctx;
|
||||
const Vector &qp_cache;
|
||||
inputs_t inputs;
|
||||
outputs_t outputs;
|
||||
const bool use_sum_factorization;
|
||||
const std::vector<const DofToQuad *> dtqs;
|
||||
const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
|
||||
const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
|
||||
const std::array<bool, n_inputs> input_is_dependent;
|
||||
const size_t trial_field_uf;
|
||||
const size_t test_field_uf;
|
||||
const ParFiniteElementSpace *test_fes;
|
||||
const ParFiniteElementSpace *trial_fes;
|
||||
const int test_vdim;
|
||||
/// Per-output row geometry of the quadrature point cache. DerivativeSetup
|
||||
/// lays that cache out over every output FieldOperator, so reading it needs
|
||||
/// all of them.
|
||||
const std::array<int, n_outputs> out_vdim;
|
||||
const std::array<int, n_outputs> out_op_dim;
|
||||
const std::array<int, n_outputs> out_offsets;
|
||||
const int output_size_on_qp;
|
||||
const int num_test_dof;
|
||||
const int trial_vdim;
|
||||
const int trial_op_dim;
|
||||
const int num_trial_dof;
|
||||
const int dim, ne, nq, q1d;
|
||||
const int num_trial_dof_1d;
|
||||
const int total_trial_op_dim;
|
||||
mutable Vector inputs_trial_op_dim;
|
||||
mutable Vector Ae_mem;
|
||||
|
||||
public:
|
||||
DerivativeAssemble() = delete;
|
||||
|
||||
DerivativeAssemble(IntegratorContext ctx_in,
|
||||
qfunc_t /*qfunc*/,
|
||||
inputs_t inputs_in,
|
||||
outputs_t outputs_in,
|
||||
const Vector &qp_cache_in):
|
||||
ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
|
||||
outputs(outputs_in), use_sum_factorization(
|
||||
[&]
|
||||
{
|
||||
const Element::Type etype =
|
||||
Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
|
||||
return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
|
||||
}()),
|
||||
dtqs(
|
||||
[&]
|
||||
{
|
||||
const DofToQuad::Mode dtq_mode = use_sum_factorization
|
||||
? DofToQuad::Mode::TENSOR
|
||||
: DofToQuad::Mode::FULL;
|
||||
std::vector<const DofToQuad *> maps;
|
||||
maps.reserve(ctx_in.unionfds.size());
|
||||
for (const auto &field : ctx_in.unionfds)
|
||||
{
|
||||
maps.emplace_back(
|
||||
GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
|
||||
}
|
||||
return maps;
|
||||
}()),
|
||||
input_dtq_maps(create_dtq_maps<Entity::Element>(
|
||||
inputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx_in, inputs),
|
||||
ctx_in.unionfds,
|
||||
ctx_in.ir)),
|
||||
output_dtq_maps(create_dtq_maps<Entity::Element>(
|
||||
outputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx_in, outputs),
|
||||
ctx_in.unionfds,
|
||||
ctx_in.ir)),
|
||||
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
|
||||
trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
|
||||
test_field_uf(
|
||||
find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
|
||||
test_fes(
|
||||
[&]
|
||||
{
|
||||
const auto *fes = std::get_if<const ParFiniteElementSpace *>(
|
||||
&ctx_in.unionfds[test_field_uf].data);
|
||||
MFEM_ASSERT(fes != nullptr && *fes != nullptr,
|
||||
"LocalQFBackend: test space is not a ParFiniteElementSpace");
|
||||
return *fes;
|
||||
}()),
|
||||
trial_fes(
|
||||
[&]
|
||||
{
|
||||
const auto *fes = std::get_if<const ParFiniteElementSpace *>(
|
||||
&ctx_in.unionfds[trial_field_uf].data);
|
||||
MFEM_ASSERT(fes != nullptr && *fes != nullptr,
|
||||
"LocalQFBackend: trial space is not a ParFiniteElementSpace");
|
||||
return *fes;
|
||||
}()),
|
||||
// All outputs are attached to the same test field, so vdim is common to
|
||||
// them; only the operator dimension differs, and that lives in out_op_dim.
|
||||
test_vdim(get<0>(outputs).vdim),
|
||||
out_vdim(get_vdim(outputs)),
|
||||
out_op_dim(compute_out_op_dim(outputs)),
|
||||
out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
|
||||
output_size_on_qp(
|
||||
[&]
|
||||
{
|
||||
int s = 0;
|
||||
for_constexpr<n_outputs>([&](auto o) { s += get<o>(outputs).size_on_qp; });
|
||||
return s;
|
||||
}()),
|
||||
num_test_dof(test_fes->GetFE(0)->GetDof()),
|
||||
trial_vdim(compute_trial_vdim(inputs, derivative_id)), trial_op_dim(
|
||||
[&]
|
||||
{
|
||||
int top = 0;
|
||||
for_constexpr<n_inputs>([&](auto i)
|
||||
{
|
||||
if (get<i>(inputs).GetFieldId() == derivative_id)
|
||||
{
|
||||
top = get<i>(inputs).size_on_qp / get<i>(inputs).vdim;
|
||||
}
|
||||
});
|
||||
return top;
|
||||
}()),
|
||||
num_trial_dof(trial_fes->GetFE(0)->GetDof()),
|
||||
dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
|
||||
nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
|
||||
num_trial_dof_1d(tensor_1d_size(num_trial_dof, dim)), total_trial_op_dim(
|
||||
[&]
|
||||
{
|
||||
const auto in_qp_sizes =
|
||||
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
|
||||
return compute_total_trial_op_dim(
|
||||
inputs, input_is_dependent, in_qp_sizes);
|
||||
}()),
|
||||
inputs_trial_op_dim(), Ae_mem()
|
||||
{
|
||||
MFEM_ASSERT(ctx.unionfds.size() == nfields,
|
||||
"LocalQFBackend: unionfds size mismatch");
|
||||
MFEM_ASSERT(trial_field_uf != SIZE_MAX,
|
||||
"DerivativeAssemble: trial field not found in unionfds");
|
||||
MFEM_ASSERT(test_field_uf != SIZE_MAX,
|
||||
"DerivativeAssemble: test field not found in unionfds");
|
||||
|
||||
MFEM_ASSERT(trial_vdim > 0,
|
||||
"LocalQFBackend: could not determine trial vdim");
|
||||
MFEM_ASSERT(total_trial_op_dim > 0,
|
||||
"LocalQFBackend: no dependent inputs found");
|
||||
|
||||
inputs_trial_op_dim.UseDevice(true);
|
||||
inputs_trial_op_dim.SetSize(n_inputs);
|
||||
auto inputs_trial_op_dim_host = inputs_trial_op_dim.HostWrite();
|
||||
for_constexpr<n_inputs>([&](auto i)
|
||||
{
|
||||
inputs_trial_op_dim_host[i] =
|
||||
input_is_dependent[i]
|
||||
? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
|
||||
: 0;
|
||||
});
|
||||
|
||||
const int elem_mat_size =
|
||||
num_test_dof * test_vdim * num_trial_dof * trial_vdim;
|
||||
Ae_mem.SetSize(elem_mat_size * ne, Device::GetDeviceMemoryType());
|
||||
Ae_mem.UseDevice(true);
|
||||
Ae_mem = 0.0;
|
||||
}
|
||||
|
||||
void operator()(SparseMatrix *&A) const
|
||||
{
|
||||
// Every output is contracted into one element matrix Ae, sized from the
|
||||
// test space of get<0>(outputs), and filled through a single test
|
||||
// ElementRestriction.
|
||||
//
|
||||
// WIP:
|
||||
// This takes care of single-field, multiple-outputs case.
|
||||
// For a multiple fields case, outputs on a second field would need a second
|
||||
// matrix -- the derivative then eould be a block column with one row block per
|
||||
// test space.
|
||||
//
|
||||
// For now we just add a check that all outputs are attached to the same test field, and abort if not.
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{
|
||||
MFEM_VERIFY(get<o>(outputs).GetFieldId() == get<0>(outputs).GetFieldId(),
|
||||
"DerivativeAssemble: every output FieldOperator has to be "
|
||||
"attached to the same test field; assembling outputs that "
|
||||
"span several fields is not supported");
|
||||
});
|
||||
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
if (!(use_sum_factorization && (dim == 2 || dim == 3)))
|
||||
{
|
||||
MFEM_ABORT("DerivativeAssemble optimized path is implemented "
|
||||
"for tensor-product 2D/3D elements only");
|
||||
}
|
||||
|
||||
DerivativeAssembleHO::Run(dim,
|
||||
q1d,
|
||||
ctx,
|
||||
qp_cache,
|
||||
Ae_mem,
|
||||
inputs,
|
||||
outputs,
|
||||
input_dtq_maps,
|
||||
output_dtq_maps,
|
||||
out_vdim,
|
||||
out_op_dim,
|
||||
out_offsets,
|
||||
output_size_on_qp,
|
||||
inputs_trial_op_dim,
|
||||
test_vdim,
|
||||
num_test_dof,
|
||||
num_trial_dof,
|
||||
num_trial_dof_1d,
|
||||
trial_vdim,
|
||||
total_trial_op_dim,
|
||||
nq,
|
||||
ne,
|
||||
q1d,
|
||||
dim);
|
||||
|
||||
A = new SparseMatrix;
|
||||
A->OverrideSize(test_fes->GetVSize(), trial_fes->GetVSize());
|
||||
|
||||
const auto *test_restr = dynamic_cast<const ElementRestriction *>(
|
||||
test_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
|
||||
const auto *trial_restr = dynamic_cast<const ElementRestriction *>(
|
||||
trial_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
|
||||
MFEM_VERIFY(test_restr != nullptr && trial_restr != nullptr,
|
||||
"DerivativeAssemble SparseMatrix assembly requires "
|
||||
"H1/conforming ElementRestriction spaces");
|
||||
|
||||
test_restr->FillSparseMatrix(Ae_mem, *A, *trial_restr);
|
||||
}
|
||||
|
||||
template<typename backend_t = LocalQFHOBackend<3>, int T_Q1D = 0>
|
||||
static void derivative_assemble_callback(
|
||||
const IntegratorContext &ctx,
|
||||
const Vector &qp_cache,
|
||||
Vector &Ae_mem,
|
||||
const inputs_t &inputs,
|
||||
const outputs_t &outputs,
|
||||
const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
|
||||
const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
|
||||
const std::array<int, n_outputs> &out_vdim,
|
||||
const std::array<int, n_outputs> &out_op_dim,
|
||||
const std::array<int, n_outputs> &out_offsets,
|
||||
const int output_size_on_qp,
|
||||
const Vector &inputs_trial_op_dim,
|
||||
const int test_vdim,
|
||||
const int num_test_dof,
|
||||
const int num_trial_dof,
|
||||
const int num_trial_dof_1d,
|
||||
const int trial_vdim,
|
||||
const int total_trial_op_dim,
|
||||
const int nq,
|
||||
const int ne,
|
||||
const int q1d,
|
||||
const int dim)
|
||||
{
|
||||
static constexpr int DIM = backend_t::DIM;
|
||||
static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
|
||||
static constexpr int MNQ = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
|
||||
|
||||
MFEM_VERIFY(dim == DIM,
|
||||
"DerivativeAssemble: mesh dim does not match backend");
|
||||
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
|
||||
MFEM_VERIFY(q1d <= MQ1, "q1d exceeds backend MQ1 limit");
|
||||
MFEM_VERIFY(nq <= MNQ,
|
||||
"DerivativeAssemble: nq exceeds backend quadrature capacity");
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{
|
||||
MFEM_VERIFY(out_op_dim[o] <= DIM,
|
||||
"DerivativeAssemble: test_op_dim exceeds spatial DIM");
|
||||
});
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
const auto d_attr = ctx.attr.Read();
|
||||
const bool has_attr = ctx.attr.Size() > 0;
|
||||
const auto d_elem_attr = ctx.elem_attr->Read();
|
||||
|
||||
const auto qpdc = Reshape(qp_cache.Read(),
|
||||
nq,
|
||||
total_trial_op_dim,
|
||||
trial_vdim,
|
||||
output_size_on_qp,
|
||||
ne);
|
||||
const auto itod = Reshape(inputs_trial_op_dim.Read(), n_inputs);
|
||||
|
||||
auto Ae = Reshape(Ae_mem.ReadWrite(),
|
||||
num_test_dof,
|
||||
test_vdim,
|
||||
num_trial_dof,
|
||||
trial_vdim,
|
||||
ne);
|
||||
|
||||
dfem::forall(
|
||||
[=] MFEM_HOST_DEVICE(const int e, void *)
|
||||
{
|
||||
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
|
||||
|
||||
static constexpr int DIM = backend_t::DIM;
|
||||
static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
|
||||
|
||||
static constexpr int MQN = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
|
||||
static constexpr int fhat_slab_size = MQN * 4;
|
||||
|
||||
MFEM_SHARED typename backend_t::Shared s;
|
||||
// One slab shared by every output. Declaring it inside the templated
|
||||
// per-output kernel allocates one per output instead, and static
|
||||
// shared memory is summed across instantiations on device.
|
||||
MFEM_SHARED real_t fhat_storage[fhat_slab_size];
|
||||
|
||||
// Each output contributes its own rows of the cache, contracted
|
||||
// against its own test basis operation; map_quadrature_data_to_fields
|
||||
// accumulates, so the element matrix is the sum over outputs for the
|
||||
// same field.
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{
|
||||
using output_fop_t = std::decay_t<decltype(get<o>(outputs))>;
|
||||
|
||||
if constexpr (!is_identity_fop_v<output_fop_t>)
|
||||
{
|
||||
// The outputs share fhat_storage, so one has to be done with it
|
||||
// before the next zeroes it.
|
||||
MFEM_SYNC_THREAD;
|
||||
detail::assemble_element_mat_sumfact<DIM, MQ1>(Ae,
|
||||
qpdc,
|
||||
e,
|
||||
itod,
|
||||
inputs,
|
||||
get<o>(outputs),
|
||||
input_dtq_maps,
|
||||
output_dtq_maps[o],
|
||||
out_offsets[o],
|
||||
out_vdim[o],
|
||||
out_op_dim[o],
|
||||
q1d,
|
||||
num_trial_dof_1d,
|
||||
fhat_storage,
|
||||
s);
|
||||
}
|
||||
});
|
||||
},
|
||||
ne,
|
||||
backend_t::thread_blocks(q1d),
|
||||
0,
|
||||
nullptr);
|
||||
}
|
||||
|
||||
using AssembleKernelType =
|
||||
decltype(&DerivativeAssemble::derivative_assemble_callback<>);
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleHO,
|
||||
AssembleKernelType,
|
||||
(int, int));
|
||||
};
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline
|
||||
typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
AssembleKernelType
|
||||
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeAssembleHO::Kernel()
|
||||
{
|
||||
static_assert(DIM == 2 || DIM == 3);
|
||||
using assemble_t =
|
||||
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return assemble_t::template derivative_assemble_callback<
|
||||
LocalQFHOBackend<DIM, Q1D>>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline
|
||||
typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
AssembleKernelType
|
||||
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeAssembleHO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using assemble_t =
|
||||
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeAssembleHO = typename assemble_t::DerivativeAssembleHO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeAssembleHO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeAssembleHO, 3, 8>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future::LocalQFImpl
|
||||
@@ -1,540 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
|
||||
#include "kernels.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
#include <array>
|
||||
|
||||
namespace mfem::future::LocalQFImpl
|
||||
{
|
||||
|
||||
// Assemble diagonal of cached Jacobian (square trial == test, tensor 2D/3D)
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
class DerivativeAssembleDiagonal
|
||||
{
|
||||
static constexpr auto inout_tuple =
|
||||
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
|
||||
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
|
||||
static constexpr size_t nfields =
|
||||
count_unique_field_ids(filtered_inout_tuple);
|
||||
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
|
||||
const IntegratorContext ctx;
|
||||
const Vector &qp_cache;
|
||||
inputs_t inputs;
|
||||
outputs_t outputs;
|
||||
const bool use_sum_factorization;
|
||||
const std::vector<const DofToQuad *> dtqs;
|
||||
const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
|
||||
const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
|
||||
const std::array<bool, n_inputs> input_is_dependent;
|
||||
const size_t trial_field_uf;
|
||||
const size_t test_field_uf;
|
||||
const bool is_square;
|
||||
const int test_vdim;
|
||||
const std::array<int, n_outputs> out_vdim;
|
||||
const std::array<int, n_outputs> out_op_dim;
|
||||
const std::array<int, n_outputs> out_offsets;
|
||||
const int output_size_on_qp;
|
||||
const int num_test_dof;
|
||||
const int num_test_dof_1d;
|
||||
const int trial_vdim;
|
||||
const int total_trial_op_dim;
|
||||
const int num_trial_dof_1d;
|
||||
const int residual_size_on_qp;
|
||||
const int dim, ne, nq, q1d;
|
||||
const std::array<int, n_inputs> inputs_trial_op_dim;
|
||||
mutable Vector Ye_mem;
|
||||
|
||||
public:
|
||||
DerivativeAssembleDiagonal() = delete;
|
||||
|
||||
DerivativeAssembleDiagonal(IntegratorContext ctx_in,
|
||||
qfunc_t /*qfunc*/,
|
||||
inputs_t inputs_in,
|
||||
outputs_t outputs_in,
|
||||
const Vector &qp_cache_in):
|
||||
ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
|
||||
outputs(outputs_in), use_sum_factorization(
|
||||
[&]
|
||||
{
|
||||
const Element::Type etype =
|
||||
Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
|
||||
return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
|
||||
}()),
|
||||
dtqs(
|
||||
[&]
|
||||
{
|
||||
const DofToQuad::Mode dtq_mode = use_sum_factorization
|
||||
? DofToQuad::Mode::TENSOR
|
||||
: DofToQuad::Mode::FULL;
|
||||
std::vector<const DofToQuad *> maps;
|
||||
maps.reserve(ctx_in.unionfds.size());
|
||||
for (const auto &field : ctx_in.unionfds)
|
||||
{
|
||||
maps.emplace_back(
|
||||
GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
|
||||
}
|
||||
return maps;
|
||||
}()),
|
||||
input_dtq_maps(create_dtq_maps<Entity::Element>(
|
||||
inputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx_in, inputs),
|
||||
ctx_in.unionfds,
|
||||
ctx_in.ir)),
|
||||
output_dtq_maps(create_dtq_maps<Entity::Element>(
|
||||
outputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx_in, outputs),
|
||||
ctx_in.unionfds,
|
||||
ctx_in.ir)),
|
||||
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
|
||||
trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
|
||||
test_field_uf(
|
||||
find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
|
||||
is_square(
|
||||
[&]
|
||||
{
|
||||
const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
|
||||
&ctx_in.unionfds[test_field_uf].data);
|
||||
const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
|
||||
&ctx_in.unionfds[trial_field_uf].data);
|
||||
return test_fes && trial_fes && *test_fes && *trial_fes &&
|
||||
(*test_fes == *trial_fes);
|
||||
}()),
|
||||
test_vdim(get<0>(outputs).vdim),
|
||||
out_vdim(get_vdim(outputs_in)),
|
||||
out_op_dim(compute_out_op_dim(outputs_in)),
|
||||
out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
|
||||
output_size_on_qp(
|
||||
[&]
|
||||
{
|
||||
int s = 0;
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{ s += get<o>(outputs_in).size_on_qp; });
|
||||
return s;
|
||||
}()), num_test_dof(
|
||||
[&]
|
||||
{
|
||||
const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
|
||||
&ctx_in.unionfds[test_field_uf].data);
|
||||
MFEM_ASSERT(test_fes != nullptr && *test_fes != nullptr,
|
||||
"LocalQFBackend: test space is not a ParFiniteElementSpace");
|
||||
return (*test_fes)->GetFE(0)->GetDof();
|
||||
}()),
|
||||
num_test_dof_1d(tensor_1d_size(num_test_dof, ctx_in.mesh.Dimension())),
|
||||
trial_vdim(compute_trial_vdim(inputs, derivative_id)), total_trial_op_dim(
|
||||
[&]
|
||||
{
|
||||
const auto input_size_on_qp =
|
||||
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
|
||||
return compute_total_trial_op_dim(
|
||||
inputs, input_is_dependent, input_size_on_qp);
|
||||
}()),
|
||||
num_trial_dof_1d(
|
||||
[&]
|
||||
{
|
||||
const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
|
||||
&ctx_in.unionfds[trial_field_uf].data);
|
||||
MFEM_ASSERT(trial_fes != nullptr && *trial_fes != nullptr,
|
||||
"LocalQFBackend: trial space is not a ParFiniteElementSpace");
|
||||
const int num_trial_dof = (*trial_fes)->GetFE(0)->GetDof();
|
||||
return tensor_1d_size(num_trial_dof, ctx_in.mesh.Dimension());
|
||||
}()),
|
||||
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
|
||||
dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
|
||||
nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
|
||||
inputs_trial_op_dim(
|
||||
[&]
|
||||
{
|
||||
std::array<int, n_inputs> itod{};
|
||||
for_constexpr<n_inputs>([&](auto i)
|
||||
{
|
||||
itod[i] = input_is_dependent[i]
|
||||
? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
|
||||
: 0;
|
||||
});
|
||||
return itod;
|
||||
}()),
|
||||
Ye_mem()
|
||||
{
|
||||
MFEM_ASSERT(ctx.unionfds.size() == nfields,
|
||||
"LocalQFBackend: unionfds size mismatch");
|
||||
MFEM_ASSERT(
|
||||
trial_field_uf != SIZE_MAX,
|
||||
"DerivativeAssembleDiagonal: trial field not found in unionfds");
|
||||
MFEM_ASSERT(
|
||||
test_field_uf != SIZE_MAX,
|
||||
"DerivativeAssembleDiagonal: test field not found in unionfds");
|
||||
MFEM_ASSERT(trial_vdim > 0,
|
||||
"LocalQFBackend: could not determine trial vdim");
|
||||
MFEM_ASSERT(total_trial_op_dim > 0,
|
||||
"LocalQFBackend: no dependent inputs found");
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{
|
||||
MFEM_CONTRACT_VAR(o);
|
||||
MFEM_ASSERT(out_vdim[o] == test_vdim,
|
||||
"DerivativeAssembleDiagonal: all outputs must share the "
|
||||
"test field vdim");
|
||||
});
|
||||
|
||||
if (is_square)
|
||||
{
|
||||
Ye_mem.SetSize(num_test_dof * test_vdim * ne);
|
||||
Ye_mem.UseDevice(true);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Backend>
|
||||
void run_kernels() const
|
||||
{
|
||||
Backend::Run(dim,
|
||||
q1d,
|
||||
ctx,
|
||||
qp_cache,
|
||||
Ye_mem,
|
||||
inputs,
|
||||
outputs,
|
||||
output_dtq_maps,
|
||||
input_dtq_maps,
|
||||
test_vdim,
|
||||
out_op_dim,
|
||||
out_offsets,
|
||||
output_size_on_qp,
|
||||
num_test_dof,
|
||||
num_test_dof_1d,
|
||||
trial_vdim,
|
||||
total_trial_op_dim,
|
||||
residual_size_on_qp,
|
||||
inputs_trial_op_dim,
|
||||
nq,
|
||||
ne,
|
||||
q1d,
|
||||
dim);
|
||||
}
|
||||
|
||||
void operator()(Vector &diag_e) const
|
||||
{
|
||||
if (!is_square) { return; }
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
if (!(use_sum_factorization && (dim == 2 || dim == 3)))
|
||||
{
|
||||
MFEM_ABORT("DerivativeAssembleDiagonal optimized path is implemented "
|
||||
"for tensor-product 2D/3D elements only");
|
||||
}
|
||||
MFEM_VERIFY(num_test_dof_1d == num_trial_dof_1d,
|
||||
"DerivativeAssembleDiagonal requires matching tensor dofs");
|
||||
MFEM_VERIFY(num_test_dof_1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
|
||||
MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
|
||||
|
||||
Ye_mem = 0.0;
|
||||
|
||||
if (q1d <= LocalQFLOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeAssembleDiagonalLO>();
|
||||
}
|
||||
else if (q1d <= LocalQFHOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeAssembleDiagonalHO>();
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
|
||||
}
|
||||
|
||||
diag_e += Ye_mem;
|
||||
}
|
||||
|
||||
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
|
||||
static void derivative_assemble_diagonal_callback(
|
||||
const IntegratorContext &ctx,
|
||||
const Vector &qp_cache,
|
||||
Vector &Ye_mem,
|
||||
const inputs_t &inputs,
|
||||
const outputs_t &outputs,
|
||||
const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
|
||||
const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
|
||||
const int test_vdim,
|
||||
const std::array<int, n_outputs> &out_op_dim,
|
||||
const std::array<int, n_outputs> &out_offsets,
|
||||
const int output_size_on_qp,
|
||||
const int num_test_dof,
|
||||
const int num_test_dof_1d,
|
||||
const int trial_vdim,
|
||||
const int total_trial_op_dim,
|
||||
const int residual_size_on_qp,
|
||||
const std::array<int, n_inputs> &inputs_trial_op_dim,
|
||||
const int nq,
|
||||
const int ne,
|
||||
const int q1d,
|
||||
const int dim)
|
||||
{
|
||||
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
static constexpr bool B2D = backend_t::DIM == 2;
|
||||
static constexpr int MTPB = backend_t::MAX_THREADS_PER_BLOCK();
|
||||
|
||||
const auto d_attr = ctx.attr.Read();
|
||||
const bool has_attr = ctx.attr.Size() > 0;
|
||||
const auto d_elem_attr = ctx.elem_attr->Read();
|
||||
|
||||
auto cache_tensor = DeviceTensor<3, const real_t>(
|
||||
qp_cache.Read(), nq, residual_size_on_qp, ne);
|
||||
const int num_dofs_per_elem = num_test_dof * test_vdim;
|
||||
auto Ye = Reshape(Ye_mem.ReadWrite(), num_dofs_per_elem, ne);
|
||||
|
||||
dfem::forall<MTPB>(
|
||||
[=] MFEM_HOST_DEVICE(const int e, void *)
|
||||
{
|
||||
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
|
||||
|
||||
// The cache is written with the quadrature index fastest, then the
|
||||
// trial op index, then the (test vdim, test op) rows of all outputs
|
||||
// stacked via out_offsets.
|
||||
auto qpdc = Reshape(&cache_tensor(0, 0, e),
|
||||
nq,
|
||||
total_trial_op_dim,
|
||||
trial_vdim,
|
||||
output_size_on_qp);
|
||||
|
||||
// Backend-owned shared scratch for the sum-factorized contraction.
|
||||
MFEM_SHARED typename backend_t::Shared s_diag;
|
||||
const int nz_dof = B2D ? 1 : num_test_dof_1d;
|
||||
|
||||
for (int vd = 0; vd < test_vdim; vd++)
|
||||
{
|
||||
auto Y = Reshape(&Ye(vd * num_test_dof, e),
|
||||
num_test_dof_1d,
|
||||
num_test_dof_1d,
|
||||
nz_dof);
|
||||
|
||||
MFEM_FOREACH_THREAD(dz_t, z, nz_dof)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(dy_t, y, num_test_dof_1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(dx_t, x, num_test_dof_1d)
|
||||
{ Y(dx_t, dy_t, dz_t) = 0.0; }
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
// Accumulate every (output o, test op k, dependent input s,
|
||||
// trial op m) block of the cached Jacobian into the diagonal via
|
||||
// the backend driver.
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{
|
||||
using test_fop_t = std::decay_t<decltype(get<o>(outputs))>;
|
||||
const auto &out_dtq = output_dtq_maps[o];
|
||||
const int test_op_dim = out_op_dim[static_cast<int>(o)];
|
||||
|
||||
// Test-basis factor along a spatial axis
|
||||
const auto eval_test =
|
||||
[&](const int k, const int axis, const int q, const int d)
|
||||
{
|
||||
const auto &B = out_dtq.B;
|
||||
const auto &G = out_dtq.G;
|
||||
if constexpr (is_value_fop<test_fop_t>::value)
|
||||
{
|
||||
return (k == 0) ? B(q, 0, d) : 0.0;
|
||||
}
|
||||
else if constexpr (is_gradient_fop<test_fop_t>::value)
|
||||
{
|
||||
return (k == axis) ? G(q, 0, d) : B(q, 0, d);
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
};
|
||||
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
const int row =
|
||||
out_offsets[static_cast<int>(o)] + vd * test_op_dim + k;
|
||||
int m_offset = 0;
|
||||
for_constexpr<n_inputs>([&](auto s)
|
||||
{
|
||||
using fop_t = std::decay_t<decltype(get<s>(inputs))>;
|
||||
const int trial_op_dim =
|
||||
inputs_trial_op_dim[static_cast<int>(s)];
|
||||
if (trial_op_dim == 0) { return; }
|
||||
|
||||
const auto &in_dtq = input_dtq_maps[s];
|
||||
const auto eval_input =
|
||||
[&](const int m, const int axis, const int q,
|
||||
const int d)
|
||||
{
|
||||
if constexpr (is_value_fop<fop_t>::value)
|
||||
{
|
||||
return (m == 0) ? in_dtq.B(q, 0, d) : 0.0;
|
||||
}
|
||||
else if constexpr (is_gradient_fop<fop_t>::value)
|
||||
{
|
||||
return (m == axis) ? in_dtq.G(q, 0, d)
|
||||
: in_dtq.B(q, 0, d);
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
};
|
||||
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
const int col = m_offset + m;
|
||||
backend_t::DiagContract(
|
||||
s_diag,
|
||||
num_test_dof_1d,
|
||||
q1d,
|
||||
nz_dof,
|
||||
[&](int axis, int q, int d)
|
||||
{ return eval_test(k, axis, q, d); },
|
||||
[&](int axis, int q, int d)
|
||||
{ return eval_input(m, axis, q, d); },
|
||||
[&](int q) { return qpdc(q, col, vd, row); },
|
||||
[&](int dx, int dy, int dz, real_t u)
|
||||
{ Y(dx, dy, dz) += u; });
|
||||
}
|
||||
m_offset += trial_op_dim;
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
ne,
|
||||
backend_t::thread_blocks(std::max(q1d, num_test_dof_1d)),
|
||||
0,
|
||||
nullptr);
|
||||
}
|
||||
|
||||
using DiagonalKernelType =
|
||||
decltype(&DerivativeAssembleDiagonal::
|
||||
derivative_assemble_diagonal_callback<>);
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalLO,
|
||||
DiagonalKernelType,
|
||||
(int, int) );
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalHO,
|
||||
DiagonalKernelType,
|
||||
(int, int) );
|
||||
};
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeAssembleDiagonal<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::DiagonalKernelType
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeAssembleDiagonalLO::Kernel()
|
||||
{
|
||||
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
|
||||
using diag_t =
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return diag_t::template derivative_assemble_diagonal_callback<
|
||||
LocalQFLOBackend<DIM, Q1D>>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeAssembleDiagonal<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::DiagonalKernelType
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeAssembleDiagonalLO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using diag_t =
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeAssembleDiagonalLO =
|
||||
typename diag_t::DerivativeAssembleDiagonalLO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeAssembleDiagonal<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::DiagonalKernelType
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeAssembleDiagonalHO::Kernel()
|
||||
{
|
||||
using diag_t =
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return diag_t::template derivative_assemble_diagonal_callback<
|
||||
LocalQFHOBackend<DIM>,
|
||||
Q1D>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeAssembleDiagonal<derivative_id,
|
||||
qfunc_t,
|
||||
inputs_t,
|
||||
outputs_t>::DiagonalKernelType
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeAssembleDiagonalHO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using diag_t =
|
||||
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeAssembleDiagonalHO =
|
||||
typename diag_t::DerivativeAssembleDiagonalHO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future::LocalQFImpl
|
||||
@@ -1,600 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
|
||||
#include "kernels.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
#include <array>
|
||||
|
||||
namespace mfem::future::LocalQFImpl
|
||||
{
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
class DerivativeSetup
|
||||
{
|
||||
static constexpr auto inout_tuple =
|
||||
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
|
||||
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
|
||||
static constexpr size_t nfields =
|
||||
count_unique_field_ids(filtered_inout_tuple);
|
||||
|
||||
using qf_signature = typename get_function_signature<qfunc_t>::type;
|
||||
using qf_param_ts = typename qf_signature::parameter_ts;
|
||||
using args_tuple_t = decay_tuple<qf_param_ts>;
|
||||
|
||||
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
|
||||
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
|
||||
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
|
||||
"LocalQF: q-function arity must match inputs + outputs");
|
||||
|
||||
qfunc_t qfunc;
|
||||
const inputs_t inputs;
|
||||
const outputs_t outputs;
|
||||
const IntegratorContext ctx;
|
||||
Vector &qp_cache;
|
||||
const std::vector<const DofToQuad *> dtqs;
|
||||
// inputs: dtq, idx, B, G, d1d, q1d, vdim
|
||||
const std::array<DofToQuadMap, n_inputs> input_dtq;
|
||||
const std::array<size_t, n_inputs> input_idx;
|
||||
const std::array<const real_t *, n_inputs> input_B, input_G;
|
||||
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
|
||||
// Jacobian cache metadata
|
||||
const std::array<bool, n_inputs> input_is_dependent;
|
||||
const std::array<int, n_inputs> input_size_on_qp;
|
||||
const std::array<int, n_outputs> out_vdim;
|
||||
const std::array<int, n_outputs> out_op_dim;
|
||||
const std::array<int, n_outputs> out_offsets;
|
||||
const int output_size_on_qp;
|
||||
const int trial_vdim;
|
||||
const int total_trial_op_dim;
|
||||
const int residual_size_on_qp;
|
||||
// other constants
|
||||
const int dim, ne, nq, q1d;
|
||||
|
||||
public:
|
||||
//////////////////////////////////////////////////////////////////
|
||||
DerivativeSetup() = delete;
|
||||
|
||||
DerivativeSetup(IntegratorContext ctx,
|
||||
qfunc_t qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
Vector &qp_cache):
|
||||
qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
|
||||
qp_cache(qp_cache), dtqs(make_dtqs(ctx)),
|
||||
input_dtq(create_dtq_maps<Entity::Element>(
|
||||
inputs,
|
||||
dtqs,
|
||||
create_union_field_map_for_dtq(ctx, inputs),
|
||||
ctx.unionfds,
|
||||
ctx.ir)),
|
||||
input_idx(create_input_vector_map(ctx, inputs)),
|
||||
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
|
||||
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
|
||||
input_vdim(get_vdim(inputs)),
|
||||
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
|
||||
input_size_on_qp(
|
||||
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
|
||||
out_vdim(get_vdim(outputs)), out_op_dim(compute_out_op_dim(outputs)),
|
||||
out_offsets(compute_out_offsets(out_vdim, out_op_dim)), output_size_on_qp(
|
||||
[&]
|
||||
{
|
||||
int s = 0;
|
||||
for_constexpr<n_outputs>([&](auto o)
|
||||
{ s += get<o>(outputs).size_on_qp; });
|
||||
return s;
|
||||
}()),
|
||||
trial_vdim(compute_trial_vdim(inputs, derivative_id)),
|
||||
total_trial_op_dim(compute_total_trial_op_dim(
|
||||
inputs, input_is_dependent, input_size_on_qp)),
|
||||
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
|
||||
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
|
||||
q1d(tensor_1d_size(nq, dim))
|
||||
{
|
||||
MFEM_ASSERT(ctx.unionfds.size() == nfields,
|
||||
"LocalQFBackend: unionfds size mismatch");
|
||||
|
||||
qp_cache.SetSize(ne * nq * residual_size_on_qp);
|
||||
qp_cache.UseDevice(true);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
void operator()(const std::vector<Vector *> &xe)
|
||||
{
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
// Quadrature index is fastest-varying so that adjacent threads (one per
|
||||
// quadrature point) touch adjacent addresses.
|
||||
auto cache_tensor = DeviceTensor<3, real_t>(
|
||||
qp_cache.ReadWrite(), nq, residual_size_on_qp, ne);
|
||||
|
||||
if (q1d <= LocalQFLOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeSetupLO>(xe, cache_tensor);
|
||||
}
|
||||
else if (q1d <= LocalQFHOBackendMQ1())
|
||||
{
|
||||
run_kernels<DerivativeSetupHO>(xe, cache_tensor);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename Backend>
|
||||
void run_kernels(const std::vector<Vector *> &xe,
|
||||
DeviceTensor<3, real_t> &cache_tensor)
|
||||
{
|
||||
Backend::Run(dim,
|
||||
q1d,
|
||||
ctx,
|
||||
qfunc,
|
||||
// inputs
|
||||
input_idx,
|
||||
input_B,
|
||||
input_G,
|
||||
input_vdim,
|
||||
input_d1d,
|
||||
input_q1d,
|
||||
input_size_on_qp,
|
||||
input_is_dependent,
|
||||
// outputs / cache metadata
|
||||
out_vdim,
|
||||
out_op_dim,
|
||||
out_offsets,
|
||||
trial_vdim,
|
||||
total_trial_op_dim,
|
||||
residual_size_on_qp,
|
||||
// vectors
|
||||
xe,
|
||||
cache_tensor,
|
||||
// fallback arguments
|
||||
dim,
|
||||
q1d);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
/// Zeroes the q-function *output* slots of an argument tuple.
|
||||
///
|
||||
/// The argument tuple is built once per quadrature point and reused for
|
||||
/// every trial seed. The q-function writes through its output parameters,
|
||||
/// so those slots have to be restored before each call to give every seed
|
||||
/// the same starting state a freshly value-initialized tuple would.
|
||||
static MFEM_HOST_DEVICE inline void reset_output_args(args_tuple_t &args)
|
||||
{
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t ao = n_inputs + oc.value;
|
||||
auto &oarg = get<ao>(args);
|
||||
oarg = std::remove_reference_t<decltype(oarg)> {};
|
||||
});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
|
||||
static void
|
||||
derivative_setup_callback(const IntegratorContext &ctx,
|
||||
qfunc_t &qfunc,
|
||||
// inputs: idx, B, G, vdim, d1d, q1d
|
||||
const std::array<size_t, n_inputs> &in_idx,
|
||||
const std::array<const real_t *, n_inputs> in_B,
|
||||
const std::array<const real_t *, n_inputs> in_G,
|
||||
const std::array<int, n_inputs> &in_vdim,
|
||||
const std::array<int, n_inputs> &in_d1d,
|
||||
const std::array<int, n_inputs> &in_q1d,
|
||||
const std::array<int, n_inputs> &in_size_on_qp,
|
||||
const std::array<bool, n_inputs> &input_dep,
|
||||
// outputs / cache metadata
|
||||
const std::array<int, n_outputs> &out_vdim,
|
||||
const std::array<int, n_outputs> &out_op_dim,
|
||||
const std::array<int, n_outputs> &out_offsets,
|
||||
const int trial_vdim,
|
||||
const int total_trial_op_dim,
|
||||
const int residual_size_on_qp,
|
||||
const std::vector<Vector *> &xe,
|
||||
DeviceTensor<3, real_t> &cache_tensor,
|
||||
// fallback arguments
|
||||
const int dim,
|
||||
const int q1d)
|
||||
{
|
||||
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
|
||||
if (ctx.attr.Size() == 0) { return; }
|
||||
|
||||
static constexpr auto B2D = backend_t::DIM == 2;
|
||||
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
|
||||
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
|
||||
|
||||
const int ne = ctx.nentities;
|
||||
MFEM_CONTRACT_VAR(residual_size_on_qp);
|
||||
|
||||
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
|
||||
|
||||
// --------------------------------------------------
|
||||
// INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
|
||||
// --------------------------------------------------
|
||||
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const size_t k = in_idx[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
|
||||
in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
|
||||
}
|
||||
else if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
|
||||
in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
|
||||
"tensor-product IR expected");
|
||||
in_XE[i] = Reshape(
|
||||
ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
const auto d_attr = ctx.attr.Read();
|
||||
const bool has_attr = ctx.attr.Size() > 0;
|
||||
const auto d_elem_attr = ctx.elem_attr->Read();
|
||||
|
||||
dfem::forall<MTPB>(
|
||||
[=] MFEM_HOST_DEVICE(const int e, void *)
|
||||
{
|
||||
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
|
||||
|
||||
// -----------------------------------------------
|
||||
// Inputs argument registers + shared memory
|
||||
// -----------------------------------------------
|
||||
input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
|
||||
MFEM_SHARED typename backend_t::Shared smem;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Load primal inputs (rargs) once for this element
|
||||
// -----------------------------------------------
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
const auto &XE = in_XE[i];
|
||||
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
|
||||
const real_t *B = in_B[i], *G = in_G[i];
|
||||
auto &rarg = get<i>(rargs);
|
||||
using XE_t = decltype(XE);
|
||||
using rarg_t = decltype(rarg);
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
if constexpr (is_value_fop<FOP>::value)
|
||||
{
|
||||
backend_t::template LoadValue<rarg_t, XE_t>(
|
||||
smem, e, d, q, Q1D, B, XE, rarg);
|
||||
}
|
||||
else if constexpr (is_gradient_fop_v<FOP>)
|
||||
{
|
||||
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
|
||||
using FieldParamT =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
backend_t::template LoadGradient<RNK, rarg_t, XE_t, FieldParamT>(
|
||||
smem, e, d, q, q1d, B, G, XE, rarg);
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
|
||||
{
|
||||
// qp values are read directly from in_XE / IR
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
// -----------------------------------------------
|
||||
// Build the primal arguments once per quadrature point, then, for
|
||||
// each trial seed (j, dependent input s, m), differentiate the
|
||||
// q-function with a unit tangent and store the result row in the
|
||||
// cache. Nothing in the primal pull depends on the seed, so the
|
||||
// thread loop is the outermost one here. The seed loops only touch
|
||||
// per-thread state, hence no barrier inside them.
|
||||
// Warning: no 'DIRECT' on the 'Z' direction,
|
||||
// as one backend may need to iterate over it.
|
||||
// -----------------------------------------------
|
||||
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
args_tuple_t primal_args {};
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
auto &parg = get<i>(primal_args);
|
||||
const auto &XE = in_XE[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
parg = XE(qx, qy, qz, 0, 0);
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
parg = backend_t::template qp_pull<ARG>(
|
||||
get<i>(rargs), qx, qy, qz);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
int m_offset = 0;
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if (!input_dep[s]) { return; }
|
||||
|
||||
const int vdim_s = in_vdim[s];
|
||||
const int op_dim_s = in_size_on_qp[s] / vdim_s;
|
||||
|
||||
for (int m = 0; m < op_dim_s; m++)
|
||||
{
|
||||
const int col_m = m + m_offset;
|
||||
|
||||
// Enzyme writes through the output slots of the
|
||||
// primal tuple, so they are reset per seed.
|
||||
reset_output_args(primal_args);
|
||||
|
||||
args_tuple_t shadow_args {};
|
||||
qf_set_value_at(get<s>(shadow_args), j, m, 1.0);
|
||||
|
||||
call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
|
||||
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value, ao = n_inputs + o;
|
||||
const auto &tangent = get<ao>(shadow_args);
|
||||
const int tv = out_vdim[o], to = out_op_dim[o];
|
||||
for (int i = 0; i < tv; i++)
|
||||
{
|
||||
for (int k = 0; k < to; k++)
|
||||
{
|
||||
const int row = out_offsets[o] + i * to + k;
|
||||
const int cache_idx =
|
||||
row * trial_vdim * total_trial_op_dim +
|
||||
j * total_trial_op_dim + col_m;
|
||||
cache_tensor(q, cache_idx, e) =
|
||||
qf_value_at(tangent, i, k);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
m_offset += op_dim_s;
|
||||
});
|
||||
}
|
||||
#else // MFEM_USE_ENZYME
|
||||
args_tuple_t qargs {};
|
||||
for_constexpr<n_inputs>([&](auto ic)
|
||||
{
|
||||
constexpr size_t i = ic.value;
|
||||
auto &qarg = get<i>(qargs);
|
||||
const auto &XE = in_XE[i];
|
||||
using FOP = tuple_element_t<i, inputs_t>;
|
||||
using ARG =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
|
||||
if constexpr (is_identity_fop_v<FOP>)
|
||||
{
|
||||
using DT =
|
||||
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
qarg = backend_t::template identity_qp_pull_dual<DT>(
|
||||
false, XE, XE, qx, qy, qz, e);
|
||||
}
|
||||
else
|
||||
{
|
||||
qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
|
||||
}
|
||||
}
|
||||
else if constexpr (is_weight_fop_v<FOP>)
|
||||
{
|
||||
qarg = XE(qx, qy, qz, 0, 0);
|
||||
}
|
||||
else if constexpr (is_value_fop_v<FOP> ||
|
||||
is_gradient_fop_v<FOP>)
|
||||
{
|
||||
qarg = backend_t::template qp_pull<ARG>(
|
||||
get<i>(rargs), qx, qy, qz);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
});
|
||||
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
int m_offset = 0;
|
||||
for_constexpr<n_inputs>([&](auto sc)
|
||||
{
|
||||
constexpr size_t s = sc.value;
|
||||
if (!input_dep[s]) { return; }
|
||||
|
||||
const int vdim_s = in_vdim[s];
|
||||
const int op_dim_s = in_size_on_qp[s] / vdim_s;
|
||||
|
||||
for (int m = 0; m < op_dim_s; m++)
|
||||
{
|
||||
const int col_m = m + m_offset;
|
||||
|
||||
// The q-function writes through the output slots,
|
||||
// so they are reset per seed.
|
||||
reset_output_args(qargs);
|
||||
|
||||
qf_set_gradient_at(get<s>(qargs), j, m, 1.0);
|
||||
|
||||
call_qfunc_no_move(qfunc, qargs);
|
||||
|
||||
for_constexpr<n_outputs>([&](auto oc)
|
||||
{
|
||||
constexpr size_t o = oc.value, ao = n_inputs + o;
|
||||
const auto &tangent = get<ao>(qargs);
|
||||
const int tv = out_vdim[o], to = out_op_dim[o];
|
||||
for (int i = 0; i < tv; i++)
|
||||
{
|
||||
for (int k = 0; k < to; k++)
|
||||
{
|
||||
const int row = out_offsets[o] + i * to + k;
|
||||
const int cache_idx =
|
||||
row * trial_vdim * total_trial_op_dim +
|
||||
j * total_trial_op_dim + col_m;
|
||||
cache_tensor(q, cache_idx, e) =
|
||||
qf_gradient_at(tangent, i, k);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Clear the seed so the next direction starts from
|
||||
// the pristine (zero-tangent) primal state.
|
||||
qf_set_gradient_at(get<s>(qargs), j, m, 0.0);
|
||||
}
|
||||
m_offset += op_dim_s;
|
||||
});
|
||||
}
|
||||
#endif // MFEM_USE_ENZYME
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
ne,
|
||||
backend_t::thread_blocks(
|
||||
compute_kernel_thread_1d<inputs_t>(q1d, in_d1d)),
|
||||
0,
|
||||
nullptr);
|
||||
}
|
||||
|
||||
using SetupKernelType =
|
||||
decltype(&DerivativeSetup::derivative_setup_callback<>);
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupLO,
|
||||
SetupKernelType,
|
||||
(int, int) );
|
||||
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupHO,
|
||||
SetupKernelType,
|
||||
(int, int) );
|
||||
};
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
SetupKernelType
|
||||
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeSetupLO::Kernel()
|
||||
{
|
||||
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
|
||||
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return setup_t::template derivative_setup_callback<
|
||||
LocalQFLOBackend<DIM, Q1D>>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
SetupKernelType
|
||||
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeSetupLO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeSetupLO = typename setup_t::DerivativeSetupLO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeSetupLO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchLOKernelByQ1D<DerivativeSetupLO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
template<int DIM, int Q1D>
|
||||
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
SetupKernelType
|
||||
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeSetupHO::Kernel()
|
||||
{
|
||||
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
return setup_t::template derivative_setup_callback<LocalQFHOBackend<DIM>,
|
||||
Q1D>;
|
||||
}
|
||||
|
||||
template<int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
SetupKernelType
|
||||
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
|
||||
DerivativeSetupHO::Fallback(int dim, int q1d)
|
||||
{
|
||||
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
|
||||
using DerivativeSetupHO = typename setup_t::DerivativeSetupHO;
|
||||
if (dim == 2)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeSetupHO, 2>(q1d);
|
||||
}
|
||||
else if (dim == 3)
|
||||
{
|
||||
return DispatchHOKernelByQ1D<DerivativeSetupHO, 3>(q1d);
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT("Unsupported dimension");
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future::LocalQFImpl
|
||||
@@ -1,277 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../util.hpp"
|
||||
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
template <typename T> struct function_traits;
|
||||
|
||||
template <typename R, typename C, typename... Args>
|
||||
struct function_traits<R (C::*)(Args...) const>
|
||||
{
|
||||
using primal_return_type = R;
|
||||
using args_tuple = std::tuple<Args...>;
|
||||
static constexpr size_t arity = sizeof...(Args);
|
||||
};
|
||||
|
||||
// Component count and writable counterpart of a per-point argument, which
|
||||
// is either a tensor or a plain scalar.
|
||||
template <typename Arg> struct qp_traits
|
||||
{
|
||||
static_assert(std::is_arithmetic_v<Arg>,
|
||||
"per-point arguments must be tensors or scalars");
|
||||
using view_type = Arg;
|
||||
static constexpr int components = 1;
|
||||
};
|
||||
|
||||
template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
|
||||
{
|
||||
using view_type = tensor<std::remove_const_t<T>, Sizes...>;
|
||||
static constexpr int components = (Sizes * ... * 1);
|
||||
};
|
||||
|
||||
// Generic FwdDiff: computes the full gradient of a pointwise qfunction at a
|
||||
// single quadrature point.
|
||||
//
|
||||
// active_input is the index of the argument to differentiate with respect
|
||||
// to; active_output is the index of the (scalar) output argument whose
|
||||
// derivative is taken.
|
||||
//
|
||||
// operator()(args...) takes the qfunction's arguments, except that the
|
||||
// active output position receives the *gradient*, shaped like the active
|
||||
// input (d(output)/d(input component)).
|
||||
//
|
||||
// Per input component d, one enzyme fwddiff call with the one-hot seed e_d
|
||||
// in the input tangent yields gradient entry d, i.e. grad_components enzyme
|
||||
// calls per point. The output tangent is not pre-zeroed, so the qfunction
|
||||
// must fully write its output (tangent stores overwrite).
|
||||
//
|
||||
// operator() is MFEM_HOST_DEVICE, stateless and allocation-free, so it is
|
||||
// callable inside a GPU kernel: seed, primal scratch and tangent all live
|
||||
// on the stack.
|
||||
template <typename Func, size_t active_input, size_t active_output>
|
||||
struct FwdDiff
|
||||
{
|
||||
using traits = function_traits<decltype(&Func::operator())>;
|
||||
using args_tuple = typename traits::args_tuple;
|
||||
static constexpr size_t arity = traits::arity;
|
||||
|
||||
static_assert(std::is_void_v<typename traits::primal_return_type>,
|
||||
"FwdDiff only supports primal functions with void return type");
|
||||
static_assert(active_input < arity && active_output < arity,
|
||||
"active argument indices must be within the function arity");
|
||||
static_assert(active_input != active_output,
|
||||
"active input and output must be different arguments");
|
||||
|
||||
using input_type =
|
||||
std::decay_t<std::tuple_element_t<active_input, args_tuple>>;
|
||||
using output_type =
|
||||
std::decay_t<std::tuple_element_t<active_output, args_tuple>>;
|
||||
|
||||
using grad_type = typename qp_traits<input_type>::view_type;
|
||||
using output_view = typename qp_traits<output_type>::view_type;
|
||||
|
||||
static constexpr int grad_components = qp_traits<input_type>::components;
|
||||
|
||||
static_assert(qp_traits<output_type>::components == 1,
|
||||
"gradient output requires a scalar output");
|
||||
|
||||
// Signature of the differentiated qfunction: the primal arguments, with
|
||||
// the Active output slot receiving the (writable) gradient instead.
|
||||
// Exposed through create_function_signature below so that
|
||||
// DifferentiableOperator can deduce the parameter types, which it cannot
|
||||
// do from the variadic operator().
|
||||
template <size_t I>
|
||||
using qf_arg_t = std::conditional_t<I == active_output, grad_type &,
|
||||
std::tuple_element_t<I, args_tuple>>;
|
||||
|
||||
template <size_t... Is>
|
||||
static FunctionSignature<void(qf_arg_t<Is>...)>
|
||||
signature_impl(std::index_sequence<Is...>);
|
||||
|
||||
using signature =
|
||||
decltype(signature_impl(std::make_index_sequence<arity> {}));
|
||||
|
||||
// d-th scalar of a per-point argument in flat row-major order, regardless
|
||||
// of rank, built on the native operator[] (tensor has no flat-index
|
||||
// accessor; flatten() returns a copy, so it cannot be written through).
|
||||
MFEM_HOST_DEVICE static double &component(double &t, int) { return t; }
|
||||
|
||||
template <typename T, int n0, int... n>
|
||||
MFEM_HOST_DEVICE static T &component(tensor<T, n0, n...> &t, int d)
|
||||
{
|
||||
if constexpr (sizeof...(n) == 0)
|
||||
{
|
||||
return t[d];
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr int stride = (n * ... * 1);
|
||||
return component(t[d / stride], d % stride);
|
||||
}
|
||||
}
|
||||
|
||||
// Plain function with the qfunction's exact (reference) signature, so it
|
||||
// can be handed to Enzyme as a function pointer; references are pointers
|
||||
// to Enzyme, so primal arguments and shadows are passed by address below.
|
||||
template <size_t... Is>
|
||||
MFEM_HOST_DEVICE static void
|
||||
static_call(std::tuple_element_t<Is, args_tuple>... args)
|
||||
{
|
||||
Func{}(args...);
|
||||
}
|
||||
|
||||
template <size_t... Is>
|
||||
static constexpr auto fn_ptr(std::index_sequence<Is...>)
|
||||
{
|
||||
return &static_call<Is...>;
|
||||
}
|
||||
static constexpr auto fn()
|
||||
{
|
||||
return fn_ptr(std::make_index_sequence<arity> {});
|
||||
}
|
||||
|
||||
// Writable, zero-initialized scratch with the shape of argument I, used
|
||||
// as its enzyme shadow.
|
||||
template <size_t I>
|
||||
using shadow_t = typename qp_traits<
|
||||
std::decay_t<std::tuple_element_t<I, args_tuple>>>::view_type;
|
||||
|
||||
template <size_t... Is>
|
||||
MFEM_HOST_DEVICE static auto make_shadows(std::index_sequence<Is...>)
|
||||
{
|
||||
return mfem::future::make_tuple(shadow_t<Is> {}...);
|
||||
}
|
||||
|
||||
template <typename Shadows, size_t... Is>
|
||||
MFEM_HOST_DEVICE static auto make_shadow_ptrs(Shadows &shadows,
|
||||
std::index_sequence<Is...>)
|
||||
{
|
||||
return mfem::future::make_tuple(&mfem::future::get<int(Is)>(shadows)...);
|
||||
}
|
||||
|
||||
// The caller's argument pointers, except the active output slot, which
|
||||
// points to scalar scratch: the caller's slot holds the gradient, while
|
||||
// the primal function writes its scalar output there.
|
||||
template <size_t I, typename Ptrs>
|
||||
MFEM_HOST_DEVICE static auto primal_ptr(Ptrs &ptrs, output_view &primal)
|
||||
{
|
||||
if constexpr (I == active_output) { return &primal; }
|
||||
else { return mfem::future::get<int(I)>(ptrs); }
|
||||
}
|
||||
|
||||
template <typename Ptrs, size_t... Is>
|
||||
MFEM_HOST_DEVICE static auto make_primal_ptrs(Ptrs &ptrs,
|
||||
output_view &primal,
|
||||
std::index_sequence<Is...>)
|
||||
{
|
||||
return mfem::future::make_tuple(primal_ptr<Is>(ptrs, primal)...);
|
||||
}
|
||||
|
||||
// Single flat enzyme call. The activity markers must appear directly in
|
||||
// the __enzyme_fwddiff argument list — Enzyme cannot trace markers that
|
||||
// were forwarded through function parameters (e.g. at -O0, where nothing
|
||||
// is inlined). Every argument is therefore enzyme_dup'd in one sticky
|
||||
// group; Const arguments simply carry a zero tangent, which is equivalent
|
||||
// to marking them enzyme_const.
|
||||
//
|
||||
// always_inline is load-bearing: when FwdDiff is itself differentiated
|
||||
// (second derivatives, forward-over-forward), Enzyme only recognizes this
|
||||
// nested __enzyme_fwddiff call if it sits at most one call level below
|
||||
// the function handed to the outer __enzyme_fwddiff. Without inlining
|
||||
// (-O0) it sits two levels down (wrapper -> operator() -> call_enzyme)
|
||||
// and the outer pass treats it as a regular call: the activity marker
|
||||
// ints then receive undef shadows, which misaligns the argument pairing
|
||||
// (observed as "cannot compute with global variable that doesn't have
|
||||
// marked shadow global" at compile time or null-shadow segfaults at
|
||||
// runtime). The always-inliner runs even at -O0, hoisting this call into
|
||||
// operator() where the nested handling applies.
|
||||
template <typename PrimalPtrs, typename ShadowPtrs, size_t... Is>
|
||||
MFEM_FUTURE_ALWAYS_INLINE
|
||||
MFEM_HOST_DEVICE static void call_enzyme(PrimalPtrs &primal_ptrs,
|
||||
ShadowPtrs &shadow_ptrs,
|
||||
std::index_sequence<Is...>)
|
||||
{
|
||||
__enzyme_fwddiff<void>(fn(), enzyme_dup,
|
||||
mfem::future::get<int(Is)>(primal_ptrs)...,
|
||||
enzyme_interleave,
|
||||
mfem::future::get<int(Is)>(shadow_ptrs)...,
|
||||
enzyme_runtime_activity);
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
MFEM_HOST_DEVICE void operator()(Args &&...args) const
|
||||
{
|
||||
static_assert(sizeof...(Args) == arity, "Wrong number of arguments");
|
||||
auto ptrs = mfem::future::make_tuple(&args...);
|
||||
|
||||
auto &grad = *mfem::future::get<int(active_output)>(ptrs);
|
||||
static_assert(std::is_same_v<std::decay_t<decltype(grad)>, grad_type>,
|
||||
"gradient argument must be shaped like the Active input "
|
||||
"(with writable scalars)");
|
||||
|
||||
constexpr auto seq = std::make_index_sequence<arity> {};
|
||||
|
||||
output_view primal{};
|
||||
auto primal_ptrs = make_primal_ptrs(ptrs, primal, seq);
|
||||
|
||||
auto shadows = make_shadows(seq);
|
||||
auto shadow_ptrs = make_shadow_ptrs(shadows, seq);
|
||||
auto &seed = mfem::future::get<int(active_input)>(shadows);
|
||||
auto &tangent = mfem::future::get<int(active_output)>(shadows);
|
||||
|
||||
// One enzyme call per input component d: seed e_d in the input tangent
|
||||
// and read gradient entry d off the output tangent.
|
||||
for (int d = 0; d < grad_components; d++)
|
||||
{
|
||||
component(seed, d) = 1.0;
|
||||
call_enzyme(primal_ptrs, shadow_ptrs, seq);
|
||||
component(grad, d) = component(tangent, 0);
|
||||
component(seed, d) = 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
static void print() { print_impl(std::make_index_sequence<arity> {}); }
|
||||
|
||||
template <size_t... Is> static void print_impl(std::index_sequence<Is...>)
|
||||
{
|
||||
mfem::out << "for d in [0, " << grad_components
|
||||
<< "): __enzyme_fwddiff<void>(fptr, enzyme_dup";
|
||||
((mfem::out << ", "
|
||||
<< get_type_name<std::tuple_element_t<Is, args_tuple>>()),
|
||||
...);
|
||||
mfem::out << ", enzyme_interleave";
|
||||
(([&]
|
||||
{
|
||||
if constexpr (Is == active_input) { mfem::out << ", e_d seed"; }
|
||||
else if constexpr (Is == active_output) { mfem::out << ", tangent out"; }
|
||||
else { mfem::out << ", zero tangent"; }
|
||||
}()),
|
||||
...);
|
||||
mfem::out << ")\n";
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Func, size_t active_input, size_t active_output>
|
||||
struct create_function_signature<FwdDiff<Func, active_input, active_output>>
|
||||
{
|
||||
using type =
|
||||
typename FwdDiff<Func, active_input, active_output>::signature;
|
||||
};
|
||||
|
||||
} // namespace mfem::future
|
||||
|
||||
#endif // MFEM_USE_ENZYME
|
||||
@@ -1,83 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
|
||||
// Explicit instantiation of the local Q-function backend device code
|
||||
|
||||
#include "../../../../config/config.hpp"
|
||||
|
||||
#ifdef MFEM_USE_MPI
|
||||
|
||||
#include "kernels.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Low-order backends instantiations for (DIM, Q1D)
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template struct lo_ker_backend<2, 2>;
|
||||
template struct lo_ker_backend<2, 3>;
|
||||
template struct lo_ker_backend<2, 4>;
|
||||
template struct lo_ker_backend<2, 5>;
|
||||
template struct lo_ker_backend<2, 6>;
|
||||
template struct lo_ker_backend<2, 7>;
|
||||
template struct lo_ker_backend<2, 8>;
|
||||
|
||||
template struct lo_ker_backend<3, 2>;
|
||||
template struct lo_ker_backend<3, 3>;
|
||||
template struct lo_ker_backend<3, 4>;
|
||||
template struct lo_ker_backend<3, 5>;
|
||||
template struct lo_ker_backend<3, 6>;
|
||||
template struct lo_ker_backend<3, 7>;
|
||||
template struct lo_ker_backend<3, 8>;
|
||||
|
||||
template struct LocalQFLOBackend<2, 2>;
|
||||
template struct LocalQFLOBackend<2, 3>;
|
||||
template struct LocalQFLOBackend<2, 4>;
|
||||
template struct LocalQFLOBackend<2, 5>;
|
||||
template struct LocalQFLOBackend<2, 6>;
|
||||
template struct LocalQFLOBackend<2, 7>;
|
||||
template struct LocalQFLOBackend<2, 8>;
|
||||
|
||||
template struct LocalQFLOBackend<3, 2>;
|
||||
template struct LocalQFLOBackend<3, 3>;
|
||||
template struct LocalQFLOBackend<3, 4>;
|
||||
template struct LocalQFLOBackend<3, 5>;
|
||||
template struct LocalQFLOBackend<3, 6>;
|
||||
template struct LocalQFLOBackend<3, 7>;
|
||||
template struct LocalQFLOBackend<3, 8>;
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// High-order backends instantiations for (DIM, Q1D)
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template struct ho_ker_backend<2, 8>;
|
||||
template struct ho_ker_backend<2, 10>;
|
||||
template struct ho_ker_backend<2, 12>;
|
||||
template struct ho_ker_backend<2, 16>;
|
||||
|
||||
template struct ho_ker_backend<3, 8>;
|
||||
template struct ho_ker_backend<3, 10>;
|
||||
template struct ho_ker_backend<3, 12>;
|
||||
template struct ho_ker_backend<3, 16>;
|
||||
|
||||
template struct LocalQFHOBackend<2, 8>;
|
||||
template struct LocalQFHOBackend<2, 10>;
|
||||
template struct LocalQFHOBackend<2, 12>;
|
||||
template struct LocalQFHOBackend<2, 16>;
|
||||
|
||||
template struct LocalQFHOBackend<3, 8>;
|
||||
template struct LocalQFHOBackend<3, 10>;
|
||||
template struct LocalQFHOBackend<3, 12>;
|
||||
template struct LocalQFHOBackend<3, 16>;
|
||||
|
||||
} // namespace mfem::future
|
||||
|
||||
#endif // MFEM_USE_MPI
|
||||
@@ -1,77 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "kernels_lo.hpp" // IWYU pragma: export
|
||||
#include "kernels_ho.hpp" // IWYU pragma: export
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Low-order backends instantiations for (DIM, Q1D)
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
extern template struct lo_ker_backend<2, 2>;
|
||||
extern template struct lo_ker_backend<2, 3>;
|
||||
extern template struct lo_ker_backend<2, 4>;
|
||||
extern template struct lo_ker_backend<2, 5>;
|
||||
extern template struct lo_ker_backend<2, 6>;
|
||||
extern template struct lo_ker_backend<2, 7>;
|
||||
extern template struct lo_ker_backend<2, 8>;
|
||||
|
||||
extern template struct lo_ker_backend<3, 2>;
|
||||
extern template struct lo_ker_backend<3, 3>;
|
||||
extern template struct lo_ker_backend<3, 4>;
|
||||
extern template struct lo_ker_backend<3, 5>;
|
||||
extern template struct lo_ker_backend<3, 6>;
|
||||
extern template struct lo_ker_backend<3, 7>;
|
||||
extern template struct lo_ker_backend<3, 8>;
|
||||
|
||||
extern template struct LocalQFLOBackend<2, 2>;
|
||||
extern template struct LocalQFLOBackend<2, 3>;
|
||||
extern template struct LocalQFLOBackend<2, 4>;
|
||||
extern template struct LocalQFLOBackend<2, 5>;
|
||||
extern template struct LocalQFLOBackend<2, 6>;
|
||||
extern template struct LocalQFLOBackend<2, 7>;
|
||||
extern template struct LocalQFLOBackend<2, 8>;
|
||||
|
||||
extern template struct LocalQFLOBackend<3, 2>;
|
||||
extern template struct LocalQFLOBackend<3, 3>;
|
||||
extern template struct LocalQFLOBackend<3, 4>;
|
||||
extern template struct LocalQFLOBackend<3, 5>;
|
||||
extern template struct LocalQFLOBackend<3, 6>;
|
||||
extern template struct LocalQFLOBackend<3, 7>;
|
||||
extern template struct LocalQFLOBackend<3, 8>;
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// High-order backends instantiations for (DIM, Q1D)
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
extern template struct ho_ker_backend<2, 8>;
|
||||
extern template struct ho_ker_backend<2, 10>;
|
||||
extern template struct ho_ker_backend<2, 12>;
|
||||
extern template struct ho_ker_backend<2, 16>;
|
||||
|
||||
extern template struct ho_ker_backend<3, 8>;
|
||||
extern template struct ho_ker_backend<3, 10>;
|
||||
extern template struct ho_ker_backend<3, 12>;
|
||||
extern template struct ho_ker_backend<3, 16>;
|
||||
|
||||
extern template struct LocalQFHOBackend<2, 8>;
|
||||
extern template struct LocalQFHOBackend<2, 10>;
|
||||
extern template struct LocalQFHOBackend<2, 12>;
|
||||
extern template struct LocalQFHOBackend<2, 16>;
|
||||
|
||||
extern template struct LocalQFHOBackend<3, 8>;
|
||||
extern template struct LocalQFHOBackend<3, 10>;
|
||||
extern template struct LocalQFHOBackend<3, 12>;
|
||||
extern template struct LocalQFHOBackend<3, 16>;
|
||||
|
||||
} // namespace mfem::future
|
||||
@@ -1,803 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../../kernels.hpp"
|
||||
namespace ker = mfem::kernels::internal;
|
||||
|
||||
#include "../../util.hpp" // for ThreadBlocks
|
||||
#include "util.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
inline constexpr int LocalQFHOBackendMQ1() { return 16; }
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
/// Register type for one HO q-function parameter
|
||||
template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
|
||||
struct ho_qreg;
|
||||
|
||||
template<typename KerOps, typename T>
|
||||
struct ho_qreg<KerOps, T, 0>
|
||||
{
|
||||
using type = typename KerOps::template val_reg_t<1>;
|
||||
};
|
||||
|
||||
template<typename KerOps, typename T>
|
||||
struct ho_qreg<KerOps, T, 1>
|
||||
{
|
||||
static constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
using type = typename KerOps::template val_reg_t<e0>;
|
||||
};
|
||||
|
||||
template<typename KerOps, typename T>
|
||||
struct ho_qreg<KerOps, T, 2>
|
||||
{
|
||||
static constexpr int VDIM = qf_param_shape<T>::extents[0];
|
||||
static constexpr int SDIM = qf_param_shape<T>::extents[1];
|
||||
using type = typename KerOps::template del_reg_t<VDIM, SDIM>;
|
||||
};
|
||||
|
||||
template<typename KerOps, typename T>
|
||||
using ho_qreg_t = typename ho_qreg<KerOps, T>::type;
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
namespace hok
|
||||
{
|
||||
|
||||
/// Load one quadrature-point value
|
||||
template<int DIM, typename T, typename Reg>
|
||||
MFEM_HOST_DEVICE inline auto load_at(Reg ®, int qx, int qy, int qz)
|
||||
{
|
||||
static_assert(DIM == 2 || DIM == 3);
|
||||
constexpr int RNK = qf_param_shape<T>::rank;
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
MFEM_CONTRACT_VAR(qz);
|
||||
if constexpr (RNK == 0) { return T{ reg(0, qy, qx) }; }
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
T t{};
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qy, qx); }
|
||||
return t;
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<T>::extents[1];
|
||||
T t;
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qy, qx); }
|
||||
}
|
||||
return t;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (RNK == 0) { return T{ reg(0, qz, qy, qx) }; }
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
T t{};
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qz, qy, qx); }
|
||||
return t;
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<T>::extents[1];
|
||||
T t;
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qz, qy, qx); }
|
||||
}
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<bool tangent, typename U>
|
||||
MFEM_HOST_DEVICE inline auto qp_store(const U &v)
|
||||
{
|
||||
if constexpr (tangent) { return qf_store_gradient(v); }
|
||||
else
|
||||
{
|
||||
return qf_store_value(v);
|
||||
}
|
||||
}
|
||||
|
||||
// Store primal value or dual tangent at one quadrature point
|
||||
template<int DIM, typename T, typename Reg, bool tangent>
|
||||
MFEM_HOST_DEVICE inline void
|
||||
store_at(Reg ®, int qx, int qy, int qz, const T &out)
|
||||
{
|
||||
static_assert(DIM == 2 || DIM == 3);
|
||||
constexpr int RNK = qf_param_shape<T>::rank;
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
MFEM_CONTRACT_VAR(qz);
|
||||
if constexpr (RNK == 0) { reg(0, qy, qx) = qp_store<tangent>(out); }
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd)
|
||||
{
|
||||
reg(dd, qy, qx) = qp_store<tangent>(out(dd));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<T>::extents[1];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j)
|
||||
{
|
||||
reg(i, j, qy, qx) = qp_store<tangent>(out(i, j));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (RNK == 0) { reg(0, qz, qy, qx) = qp_store<tangent>(out); }
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd)
|
||||
{
|
||||
reg(dd, qz, qy, qx) = qp_store<tangent>(out(dd));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<T>::extents[1];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j)
|
||||
{
|
||||
reg(i, j, qz, qy, qx) = qp_store<tangent>(out(i, j));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pull primal/tangent pair into a dual q-function argument
|
||||
template<int DIM, typename T, typename Reg>
|
||||
MFEM_HOST_DEVICE inline auto
|
||||
pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
|
||||
{
|
||||
if constexpr (!qf_param_uses_dual_v<T>)
|
||||
{
|
||||
return load_at<DIM, T>(preg, qx, qy, qz);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
|
||||
constexpr int RNK = qf_param_shape<T>::rank;
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
MFEM_CONTRACT_VAR(qz);
|
||||
if constexpr (RNK == 0)
|
||||
{
|
||||
return T{ preg(0, qy, qx), sreg(0, qy, qx) };
|
||||
}
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
T t{};
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd)
|
||||
{
|
||||
t(dd) = { preg(dd, qy, qx), sreg(dd, qy, qx) };
|
||||
}
|
||||
return t;
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<T>::extents[1];
|
||||
T t;
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j)
|
||||
{
|
||||
t(i, j) = { preg(i, j, qy, qx), sreg(i, j, qy, qx) };
|
||||
}
|
||||
}
|
||||
return t;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (RNK == 0)
|
||||
{
|
||||
return T{ preg(0, qz, qy, qx), sreg(0, qz, qy, qx) };
|
||||
}
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
T t{};
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd)
|
||||
{
|
||||
t(dd) = { preg(dd, qz, qy, qx), sreg(dd, qz, qy, qx) };
|
||||
}
|
||||
return t;
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<T>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<T>::extents[1];
|
||||
T t;
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j)
|
||||
{
|
||||
t(i, j) = { preg(i, j, qz, qy, qx), sreg(i, j, qz, qy, qx) };
|
||||
}
|
||||
}
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace hok
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
/// HO tensor-product kernels
|
||||
template<int T_DIM, int MQ1>
|
||||
struct ho_ker_backend
|
||||
{
|
||||
static constexpr int DIM = T_DIM;
|
||||
static_assert(DIM == 2 || DIM == 3);
|
||||
|
||||
template<int VDIM>
|
||||
using val_reg_t = std::conditional_t<(DIM == 2),
|
||||
ker::v_regs2d_t<VDIM, MQ1>,
|
||||
ker::v_regs3d_t<VDIM, MQ1>>;
|
||||
|
||||
template<int VDIM, int SDIM>
|
||||
using del_reg_t = std::conditional_t<(DIM == 2),
|
||||
ker::vd_regs2d_t<VDIM, SDIM, MQ1>,
|
||||
ker::vd_regs3d_t<VDIM, SDIM, MQ1>>;
|
||||
|
||||
struct Shared
|
||||
{
|
||||
real_t M[MQ1][MQ1], B[MQ1][MQ1], G[MQ1][MQ1];
|
||||
};
|
||||
|
||||
template<typename XE_t, typename Dofs>
|
||||
static MFEM_HOST_DEVICE void
|
||||
load_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
|
||||
{
|
||||
if constexpr (DIM == 2) { ker::LoadDofs2d(e, d, XE, dofs); }
|
||||
else
|
||||
{
|
||||
ker::LoadDofs3d(e, d, XE, dofs);
|
||||
}
|
||||
}
|
||||
|
||||
template<int VDIM, int SDIM, typename XE_t, typename Dofs>
|
||||
static MFEM_HOST_DEVICE void
|
||||
load_grad_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
|
||||
{
|
||||
static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
|
||||
load_dofs(e, d, XE, dofs);
|
||||
}
|
||||
|
||||
template<typename Smem, typename Dofs, typename ArgReg>
|
||||
static MFEM_HOST_DEVICE void
|
||||
eval_value(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
|
||||
{
|
||||
if constexpr (DIM == 2) { ker::Eval2d(d, q, s.M, s.B, dofs, rarg); }
|
||||
else
|
||||
{
|
||||
ker::Eval3d(d, q, s.M, s.B, dofs, rarg);
|
||||
}
|
||||
}
|
||||
|
||||
template<int VDIM, int SDIM, typename Smem, typename Dofs, typename ArgReg>
|
||||
static MFEM_HOST_DEVICE void
|
||||
grad(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
|
||||
{
|
||||
static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
|
||||
if constexpr (DIM == 2) { ker::Grad2d(d, q, s.M, s.B, s.G, dofs, rarg); }
|
||||
else
|
||||
{
|
||||
ker::Grad3d(d, q, s.M, s.B, s.G, dofs, rarg);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
|
||||
static MFEM_HOST_DEVICE void write_value(const int d,
|
||||
const int q,
|
||||
const int e,
|
||||
Smem &s,
|
||||
ArgReg &rarg,
|
||||
Dofs &dofs,
|
||||
YE_t &YE)
|
||||
{
|
||||
if constexpr (DIM == 2)
|
||||
{
|
||||
ker::EvalTranspose2d(d, q, s.M, s.B, rarg, dofs);
|
||||
ker::WriteDofs2d(e, d, dofs, YE);
|
||||
}
|
||||
else
|
||||
{
|
||||
ker::EvalTranspose3d(d, q, s.M, s.B, rarg, dofs);
|
||||
ker::WriteDofs3d(e, d, dofs, YE);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
|
||||
static MFEM_HOST_DEVICE void write_gradient_2d(const int d,
|
||||
const int q,
|
||||
const int e,
|
||||
Smem &s,
|
||||
ArgReg &rarg,
|
||||
Dofs &dofs,
|
||||
YE_t &YE)
|
||||
{
|
||||
ker::GradTranspose2d(d, q, s.M, s.B, s.G, rarg, dofs);
|
||||
ker::WriteDofs2d(e, d, dofs, YE);
|
||||
}
|
||||
|
||||
template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
|
||||
static MFEM_HOST_DEVICE void write_gradient_3d(const int d,
|
||||
const int q,
|
||||
const int e,
|
||||
Smem &s,
|
||||
ArgReg &rarg,
|
||||
Dofs &dofs,
|
||||
YE_t &YE)
|
||||
{
|
||||
ker::GradTranspose3d(d, q, s.M, s.B, s.G, rarg, dofs);
|
||||
ker::WriteDofs3d(e, d, dofs, YE);
|
||||
}
|
||||
|
||||
template<int VDIM,
|
||||
int SDIM,
|
||||
typename Smem,
|
||||
typename Dofs,
|
||||
typename ArgReg,
|
||||
typename YE_t>
|
||||
static MFEM_HOST_DEVICE void write_gradient(const int d,
|
||||
const int q,
|
||||
const int e,
|
||||
Smem &s,
|
||||
ArgReg &rarg,
|
||||
Dofs &dofs,
|
||||
YE_t &YE)
|
||||
{
|
||||
static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
|
||||
if constexpr (DIM == 2) { write_gradient_2d(d, q, e, s, rarg, dofs, YE); }
|
||||
else
|
||||
{
|
||||
write_gradient_3d(d, q, e, s, rarg, dofs, YE);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template<int T_DIM, int T_Q1D = LocalQFHOBackendMQ1()>
|
||||
struct LocalQFHOBackend
|
||||
{
|
||||
// ─────────────────────────────────────────────────────
|
||||
static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
|
||||
static_assert(DIM == 2 || DIM == 3);
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
static inline ThreadBlocks thread_blocks(const int q1d)
|
||||
{
|
||||
MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
|
||||
return { q1d, q1d, 1 };
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
static inline constexpr int MAX_THREADS_PER_BLOCK() { return Q1D * Q1D; }
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
using backend_t = ho_ker_backend<DIM, Q1D>;
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
using Shared = typename backend_t::Shared;
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename WT, typename WI, typename Cache, typename AddY>
|
||||
static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
|
||||
const int num_dof_1d,
|
||||
const int q1d,
|
||||
const int nz_dof,
|
||||
WT wt,
|
||||
WI wi,
|
||||
Cache cache,
|
||||
AddY add_y)
|
||||
{
|
||||
MFEM_CONTRACT_VAR(nz_dof);
|
||||
const int nqz = (DIM == 3) ? q1d : 1;
|
||||
const int ndz = (DIM == 3) ? num_dof_1d : 1;
|
||||
|
||||
ker::s_regs3d_t<MQ1> rz, ry;
|
||||
auto &smem = s.M;
|
||||
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
for (int dz = 0; dz < ndz; dz++)
|
||||
{
|
||||
real_t u = 0.0;
|
||||
for (int qz = 0; qz < nqz; qz++)
|
||||
{
|
||||
const int q = qx + (qy + qz * q1d) * q1d;
|
||||
const real_t wz =
|
||||
(DIM == 3) ? (wt(2, qz, dz) * wi(2, qz, dz)) : real_t(1);
|
||||
u += wz * cache(q);
|
||||
}
|
||||
rz[dz][qy][qx] = u;
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
for (int dz = 0; dz < ndz; dz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{ smem[qy][qx] = rz[dz][qy][qx]; }
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
real_t u = 0.0;
|
||||
for (int qy = 0; qy < q1d; qy++)
|
||||
{
|
||||
u += wt(1, qy, dy) * wi(1, qy, dy) * smem[qy][qx];
|
||||
}
|
||||
ry[dz][dy][qx] = u;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
|
||||
for (int dz = 0; dz < ndz; dz++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{ smem[dy][qx] = ry[dz][dy][qx]; }
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(dx, x, num_dof_1d)
|
||||
{
|
||||
real_t u = 0.0;
|
||||
for (int qx = 0; qx < q1d; qx++)
|
||||
{
|
||||
u += wt(0, qx, dx) * wi(0, qx, dx) * smem[dy][qx];
|
||||
}
|
||||
add_y(dx, dy, dz, u);
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename T>
|
||||
using QReg = ho_qreg_t<backend_t, T>;
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename ArgRegT, typename XE_T>
|
||||
static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
|
||||
const int e,
|
||||
const int d,
|
||||
const int q,
|
||||
const int,
|
||||
const real_t *B,
|
||||
const XE_T &XE,
|
||||
ArgRegT &rarg)
|
||||
{
|
||||
ker::LoadMatrix(d, q, B, s.B);
|
||||
std::remove_reference_t<ArgRegT> dofs;
|
||||
backend_t::load_dofs(e, d, XE, dofs);
|
||||
backend_t::eval_value(d, q, s, dofs, rarg);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<int RNK,
|
||||
typename ArgRegT,
|
||||
typename XE_T,
|
||||
typename FieldParamT = ArgRegT>
|
||||
static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
|
||||
const int e,
|
||||
const int d,
|
||||
const int q,
|
||||
const int,
|
||||
const real_t *B,
|
||||
const real_t *G,
|
||||
const XE_T &XE,
|
||||
ArgRegT &rarg)
|
||||
{
|
||||
ker::LoadMatrix(d, q, B, s.B);
|
||||
ker::LoadMatrix(d, q, G, s.G);
|
||||
static_assert(RNK == 1 || RNK == 2);
|
||||
static constexpr int VDIM =
|
||||
(RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
|
||||
static constexpr int SDIM = (RNK == 1)
|
||||
? qf_param_shape<FieldParamT>::extents[0]
|
||||
: qf_param_shape<FieldParamT>::extents[1];
|
||||
if constexpr (SDIM == DIM)
|
||||
{
|
||||
typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
|
||||
if constexpr (RNK == 1) { backend_t::load_dofs(e, d, XE, dofs); }
|
||||
else
|
||||
{
|
||||
backend_t::template load_grad_dofs<VDIM, SDIM>(e, d, XE, dofs);
|
||||
}
|
||||
backend_t::template grad<VDIM, SDIM>(d, q, s, dofs, rarg);
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename T>
|
||||
static MFEM_HOST_DEVICE inline auto
|
||||
qp_pull(QReg<T> ®, int qx, int qy, int qz)
|
||||
{ return hok::load_at<DIM, T>(reg, qx, qy, qz); }
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename T>
|
||||
static MFEM_HOST_DEVICE inline auto qp_pull_directional(
|
||||
QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
|
||||
{ return hok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename DT, typename XE_T>
|
||||
static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
|
||||
const XE_T &XP,
|
||||
const XE_T &XD,
|
||||
int qx,
|
||||
int qy,
|
||||
int qz,
|
||||
int e)
|
||||
{
|
||||
constexpr int RNK = qf_param_shape<DT>::rank;
|
||||
if constexpr (RNK == 0)
|
||||
{
|
||||
DT t{};
|
||||
t.value = XP(0, qx, qy, qz, e);
|
||||
t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
|
||||
return t;
|
||||
}
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<DT>::extents[0];
|
||||
DT t{};
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd)
|
||||
{
|
||||
t(dd).value = XP(dd, qx, qy, qz, e);
|
||||
t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
|
||||
}
|
||||
return t;
|
||||
}
|
||||
else if constexpr (RNK == 2)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<DT>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<DT>::extents[1];
|
||||
DT t{};
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j)
|
||||
{
|
||||
t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
|
||||
t(i, j).gradient =
|
||||
dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
|
||||
}
|
||||
}
|
||||
return t;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename T>
|
||||
static MFEM_HOST_DEVICE inline void
|
||||
qp_push(QReg<T> ®, int qx, int qy, int qz, const T &out)
|
||||
{ hok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out); }
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename T>
|
||||
static MFEM_HOST_DEVICE inline void
|
||||
qp_push_tangent(QReg<T> ®, int qx, int qy, int qz, const T &out)
|
||||
{
|
||||
hok::store_at<DIM, T, decltype(reg), qf_param_uses_dual_v<T>>(
|
||||
reg, qx, qy, qz, out);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename DT, typename YE_T>
|
||||
static MFEM_HOST_DEVICE inline void identity_qp_write_value(
|
||||
YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
|
||||
{
|
||||
constexpr int RNK = qf_param_shape<DT>::rank;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
if constexpr (RNK == 0)
|
||||
{
|
||||
YE(0, qx, qy, qz, e) = qf_store_value(qout);
|
||||
}
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<DT>::extents[0];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd)
|
||||
{
|
||||
YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
|
||||
}
|
||||
}
|
||||
else if constexpr (RNK == 2)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<DT>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<DT>::extents[1];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j)
|
||||
{
|
||||
YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename DT, typename YE_T>
|
||||
static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
|
||||
YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
|
||||
{
|
||||
constexpr int RNK = qf_param_shape<DT>::rank;
|
||||
if constexpr (qf_param_uses_dual_v<DT>)
|
||||
{
|
||||
if constexpr (RNK == 0)
|
||||
{
|
||||
YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
|
||||
}
|
||||
else if constexpr (RNK == 1)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<DT>::extents[0];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int dd = 0; dd < e0; ++dd)
|
||||
{
|
||||
YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
|
||||
}
|
||||
}
|
||||
else if constexpr (RNK == 2)
|
||||
{
|
||||
constexpr int e0 = qf_param_shape<DT>::extents[0];
|
||||
constexpr int e1 = qf_param_shape<DT>::extents[1];
|
||||
MFEM_UNROLL(e0)
|
||||
for (int i = 0; i < e0; ++i)
|
||||
{
|
||||
MFEM_UNROLL(e1)
|
||||
for (int j = 0; j < e1; ++j)
|
||||
{
|
||||
YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(false, "Unsupported");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<typename ArgRegT, typename YE_T>
|
||||
static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
|
||||
const int e,
|
||||
const int d,
|
||||
const int q,
|
||||
const int,
|
||||
const real_t *B,
|
||||
YE_T &YE,
|
||||
ArgRegT &rarg)
|
||||
{
|
||||
ker::LoadMatrix(d, q, B, s.B);
|
||||
std::remove_reference_t<ArgRegT> dofs;
|
||||
backend_t::write_value(d, q, e, s, rarg, dofs, YE);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────
|
||||
template<int RNK,
|
||||
typename ArgRegT,
|
||||
typename YE_T,
|
||||
typename FieldParamT = ArgRegT>
|
||||
static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
|
||||
const int e,
|
||||
const int d,
|
||||
const int q,
|
||||
const int,
|
||||
const real_t *B,
|
||||
const real_t *G,
|
||||
YE_T &YE,
|
||||
ArgRegT &rarg)
|
||||
{
|
||||
ker::LoadMatrix(d, q, B, s.B);
|
||||
ker::LoadMatrix(d, q, G, s.G);
|
||||
static_assert(RNK == 1 || RNK == 2);
|
||||
static constexpr int VDIM =
|
||||
(RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
|
||||
static constexpr int SDIM = (RNK == 1)
|
||||
? qf_param_shape<FieldParamT>::extents[0]
|
||||
: qf_param_shape<FieldParamT>::extents[1];
|
||||
if constexpr (SDIM == DIM)
|
||||
{
|
||||
typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
|
||||
backend_t::template write_gradient<VDIM, SDIM>(
|
||||
d, q, e, s, rarg, dofs, YE);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// @brief Dispatch to a compile-time HO kernel with MQ1 >= runtime @a q1d.
|
||||
template <typename HOKernelTable, int DIM, int MQ1 = LocalQFHOBackendMQ1()>
|
||||
inline typename HOKernelTable::KernelSignature
|
||||
DispatchHOKernelByQ1D(int q1d)
|
||||
{
|
||||
MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
|
||||
"Unsupported HO quadrature order: " << q1d);
|
||||
return HOKernelTable::template Kernel<DIM, MQ1>();
|
||||
}
|
||||
|
||||
} // namespace mfem::future
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,242 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../../integrator_ctx.hpp"
|
||||
#include "action.hpp"
|
||||
#include "derivative_action.hpp"
|
||||
#include "derivative_setup.hpp"
|
||||
#include "derivative_apply.hpp"
|
||||
#include "derivative_assemble.hpp"
|
||||
#include "derivative_assemble_diagonal.hpp"
|
||||
#include "derivative_apply_transpose.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
struct LocalQFBackend
|
||||
{
|
||||
/**
|
||||
* @brief Make an action for a local Q-function backend.
|
||||
*
|
||||
* @param ctx The integrator context.
|
||||
* @param args The arguments to the action.
|
||||
* @return The action.
|
||||
*/
|
||||
template<typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
static auto MakeAction(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs)
|
||||
{
|
||||
return LocalQFImpl::Action<qfunc_t, inputs_t, outputs_t>(
|
||||
ctx, qfunc, inputs, outputs);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Make a derivative action for a local Q-function backend.
|
||||
*
|
||||
* @tparam derivative_id The id of the derivative.
|
||||
* @param ctx The integrator context.
|
||||
* @param args The arguments to the derivative action.
|
||||
* @return The derivative action.
|
||||
*/
|
||||
template<int id, typename qfunc_t, typename inputs_t, typename outputs_t>
|
||||
static auto MakeDerivativeAction(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs)
|
||||
{
|
||||
return LocalQFImpl::DerivativeAction<id, qfunc_t, inputs_t, outputs_t>(
|
||||
ctx, qfunc, inputs, outputs);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
static auto MakeDerivativeSetup(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeSetup<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
|
||||
outputs, qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
static auto MakeDerivativeApply(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeApply<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
|
||||
outputs, qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
static auto MakeDerivativeApplyTranspose(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeApplyTranspose<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
|
||||
outputs, qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
static auto MakeDerivativeAssemble(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeAssemble<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
|
||||
outputs, qp_cache);
|
||||
}
|
||||
|
||||
template<
|
||||
int derivative_id,
|
||||
typename qfunc_t,
|
||||
typename inputs_t,
|
||||
typename outputs_t>
|
||||
static auto MakeDerivativeAssembleDiagonal(
|
||||
const IntegratorContext &ctx,
|
||||
const qfunc_t &qfunc,
|
||||
inputs_t inputs,
|
||||
outputs_t outputs,
|
||||
const Vector &qp_cache)
|
||||
{
|
||||
return LocalQFImpl::DerivativeAssembleDiagonal<
|
||||
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
|
||||
outputs, qp_cache);
|
||||
}
|
||||
};
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template<int DIM, int Q1D, typename QT, typename IT, typename OT>
|
||||
inline void AddAction()
|
||||
{
|
||||
using ker = LocalQFImpl::Action<QT, IT, OT>;
|
||||
if constexpr (Q1D <= 8)
|
||||
{
|
||||
ker::ActionLO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
else
|
||||
{
|
||||
ker::ActionHO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
|
||||
inline void AddDerivativeAction()
|
||||
{
|
||||
using ker = LocalQFImpl::DerivativeAction<DID, QT, IT, OT>;
|
||||
if constexpr (Q1D <= 8)
|
||||
{
|
||||
ker::DerivativeActionLO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
else
|
||||
{
|
||||
ker::DerivativeActionHO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
|
||||
inline void AddDerivativeSetup()
|
||||
{
|
||||
using ker = LocalQFImpl::DerivativeSetup<DID, QT, IT, OT>;
|
||||
if constexpr (Q1D <= 8)
|
||||
{
|
||||
ker::DerivativeSetupLO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
else
|
||||
{
|
||||
ker::DerivativeSetupHO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
|
||||
inline void AddDerivativeApply()
|
||||
{
|
||||
using ker = LocalQFImpl::DerivativeApply<DID, QT, IT, OT>;
|
||||
if constexpr (Q1D <= 8)
|
||||
{
|
||||
ker::DerivativeApplyLO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
else
|
||||
{
|
||||
ker::DerivativeApplyHO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
|
||||
inline void AddDerivativeApplyTranspose()
|
||||
{
|
||||
using ker = LocalQFImpl::DerivativeApplyTranspose<DID, QT, IT, OT>;
|
||||
if constexpr (Q1D <= 8)
|
||||
{
|
||||
ker::DerivativeApplyTransposeLO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
else
|
||||
{
|
||||
ker::DerivativeApplyTransposeHO::template Specialization<DIM, Q1D>::Add();
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template<int DIM, int Q1D, typename QT, typename IT, typename OT,
|
||||
typename derivative_ids_t = std::index_sequence<>>
|
||||
inline void AddLocalSpecializations()
|
||||
{
|
||||
AddAction<DIM, Q1D, QT, IT, OT>();
|
||||
|
||||
for_constexpr([&](auto i)
|
||||
{
|
||||
using derivative_id = decltype(i);
|
||||
AddDerivativeAction<DIM, Q1D, derivative_id::value, QT, IT, OT>();
|
||||
AddDerivativeSetup<DIM, Q1D, derivative_id::value, QT, IT, OT>();
|
||||
AddDerivativeApply<DIM, Q1D, derivative_id::value, QT, IT, OT>();
|
||||
AddDerivativeApplyTranspose<DIM, Q1D, derivative_id::value, QT, IT, OT>();
|
||||
}, derivative_ids_t{});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,606 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../util.hpp"
|
||||
#include "util.hpp"
|
||||
#include "../../util.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
template <typename T> struct function_traits;
|
||||
|
||||
template <typename R, typename C, typename... Args>
|
||||
struct function_traits<R (C::*)(Args...) const>
|
||||
{
|
||||
using primal_return_type = R;
|
||||
using args_tuple = tuple<Args...>;
|
||||
static constexpr size_t arity = sizeof...(Args);
|
||||
};
|
||||
|
||||
/// Scalar-level view of a per-point argument: the plain value type it stores
|
||||
/// and the dual type used to carry a first-order tangent alongside it.
|
||||
template <typename T>
|
||||
struct qp_scalar_traits
|
||||
{
|
||||
using view_type = T;
|
||||
using dual_type = dual<T, T>;
|
||||
};
|
||||
|
||||
template <typename V, typename G>
|
||||
struct qp_scalar_traits<dual<V, G>>
|
||||
{
|
||||
using view_type = V;
|
||||
using dual_type = dual<V, G>;
|
||||
};
|
||||
|
||||
// Component count and writable counterpart of a per-point argument, which
|
||||
// is either a tensor or a plain scalar. `view_type` keeps the argument's own
|
||||
// scalar type; `dual_type` is the same shape with a dual scalar, used for the
|
||||
// gradient blocks of the native dual-number backend.
|
||||
template <typename Arg> struct qp_traits
|
||||
{
|
||||
static_assert(std::is_arithmetic_v<Arg> || is_dual_number<Arg>::value,
|
||||
"per-point arguments must be tensors or scalars");
|
||||
using scalar_type = std::remove_const_t<Arg>;
|
||||
using view_type = scalar_type;
|
||||
using dual_type = typename qp_scalar_traits<scalar_type>::dual_type;
|
||||
static constexpr int components = 1;
|
||||
};
|
||||
|
||||
template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
|
||||
{
|
||||
using scalar_type = std::remove_const_t<T>;
|
||||
using view_type = tensor<scalar_type, Sizes...>;
|
||||
using dual_type =
|
||||
tensor<typename qp_scalar_traits<scalar_type>::dual_type, Sizes...>;
|
||||
static constexpr int components = (Sizes * ... * 1);
|
||||
};
|
||||
|
||||
template <typename... T1s, typename... T2s>
|
||||
constexpr tuple<T1s..., T2s...> concat_tuples(tuple<T1s...>, tuple<T2s...>);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/// Nested ("hyper") dual utilities, used for second derivatives on the native
|
||||
/// dual-number backend.
|
||||
///
|
||||
/// A second derivative taken with plain duals would have to reuse the single
|
||||
/// gradient slot that already carries the incoming direction. Lifting the
|
||||
/// scalar to `dual<dual<V,G>, dual<V,G>>` adds a second, independent slot:
|
||||
///
|
||||
/// dual(a, b) -> ((a, c), (b, d))
|
||||
///
|
||||
/// `a`/`b` stay the incoming primal/direction, `c` is seeded per component and
|
||||
/// `d` returns the second-order result.
|
||||
template <typename T>
|
||||
struct make_nested_qp_type
|
||||
{
|
||||
using type = T;
|
||||
};
|
||||
|
||||
template <typename V, typename G>
|
||||
struct make_nested_qp_type<dual<V, G>>
|
||||
{
|
||||
using type = dual<dual<V, G>, dual<V, G>>;
|
||||
};
|
||||
|
||||
template <typename S, int... Sizes>
|
||||
struct make_nested_qp_type<tensor<S, Sizes...>>
|
||||
{
|
||||
using type = tensor<typename make_nested_qp_type<S>::type, Sizes...>;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
using make_nested_qp_type_t = typename make_nested_qp_type<T>::type;
|
||||
|
||||
using native_dual_t = typename qp_scalar_traits<real_t>::dual_type;
|
||||
using nested_native_dual_t = make_nested_qp_type_t<native_dual_t>;
|
||||
|
||||
/// Rebinds a q-function's scalar template parameter so its arguments are
|
||||
/// nested duals. Only the leading scalar parameter is rebound; any remaining
|
||||
/// non-type parameters (e.g. `dim`) are carried through unchanged. This
|
||||
/// requires q-functions of the form `QFunc<scalar_t>` or `QFunc<scalar_t,
|
||||
/// Params...>`; `supported` reports whether that shape was matched, so callers
|
||||
/// can fail with a readable static_assert.
|
||||
template <typename qfunc_t, typename nested_scalar_t, typename = void>
|
||||
struct rebind_qfunc_scalar
|
||||
{
|
||||
static constexpr bool supported = false;
|
||||
};
|
||||
|
||||
template <template <typename> class qfunc_template_t,
|
||||
typename old_scalar_t,
|
||||
typename nested_scalar_t>
|
||||
struct rebind_qfunc_scalar<qfunc_template_t<old_scalar_t>, nested_scalar_t>
|
||||
{
|
||||
static constexpr bool supported = true;
|
||||
using type = qfunc_template_t<nested_scalar_t>;
|
||||
};
|
||||
|
||||
template <template <typename, auto...> class qfunc_template_t,
|
||||
typename old_scalar_t,
|
||||
auto... Params,
|
||||
typename nested_scalar_t>
|
||||
struct rebind_qfunc_scalar<qfunc_template_t<old_scalar_t, Params...>,
|
||||
nested_scalar_t,
|
||||
std::enable_if_t<(sizeof...(Params) > 0)>>
|
||||
{
|
||||
static constexpr bool supported = true;
|
||||
using type = qfunc_template_t<nested_scalar_t, Params...>;
|
||||
};
|
||||
|
||||
template <typename qfunc_t, typename nested_scalar_t>
|
||||
using rebind_qfunc_scalar_t =
|
||||
typename rebind_qfunc_scalar<qfunc_t, nested_scalar_t>::type;
|
||||
|
||||
/// Copies a q-function argument into its nested-dual counterpart, mapping
|
||||
/// dual(a, b) -> ((a, 0), (b, 0)). The inner gradients stay zero; the caller
|
||||
/// seeds one of them per component.
|
||||
template <typename Dst, typename Src>
|
||||
MFEM_HOST_DEVICE void lift_to_nested_arg(const Src &src, Dst &dst)
|
||||
{
|
||||
using dst_t = std::decay_t<Dst>;
|
||||
constexpr bool dst_uses_dual = is_dual_number<dst_t>::value ||
|
||||
qf_param_uses_dual_v<dst_t> ||
|
||||
is_nested_dual_number<dst_t>::value ||
|
||||
qf_param_uses_nested_dual_v<dst_t>;
|
||||
if constexpr (dst_uses_dual)
|
||||
{
|
||||
constexpr int ncomp = qp_traits<dst_t>::components;
|
||||
for (int component = 0; component < ncomp; component++)
|
||||
{
|
||||
qf_set_flat_value(dst, component, qf_flat_value(src, component));
|
||||
qf_set_flat_gradient(dst, component, qf_flat_gradient(src, component));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Destination carries no derivative slots: a plain copy is the lift.
|
||||
dst = src;
|
||||
}
|
||||
}
|
||||
|
||||
// RevDiff: computes the full gradient of a pointwise qfunction at a single
|
||||
// quadrature point using one Enzyme reverse-mode (autodiff) call.
|
||||
//
|
||||
// operator()(args...) takes all qfunction input primals followed by one
|
||||
// writable gradient output per Active input (same shape as that input).
|
||||
// The qfunction's own output is not passed; Enzyme writes it to stack
|
||||
// scratch (enzyme_dupnoneed).
|
||||
//
|
||||
// A single __enzyme_autodiff call with the output adjoint seeded to 1
|
||||
// yields all gradient blocks simultaneously — O(1) calls per point
|
||||
// regardless of input size, vs O(ncomp) for forward mode.
|
||||
//
|
||||
// operator() is MFEM_HOST_DEVICE and allocation-free.
|
||||
//
|
||||
// Without Enzyme the same interface is served by a forward-mode dual-number
|
||||
// fallback (`call_dual_rev`), which seeds one component at a time. `mode`
|
||||
// selects what that fallback is being asked for: `Eval` is the plain gradient,
|
||||
// `Derivative` is a gradient taken inside an outer derivative, which lifts the
|
||||
// q-function to nested duals so seeding does not clobber the outer direction.
|
||||
// With Enzyme both modes use the single reverse-mode call and `mode` is inert.
|
||||
enum class RevDiffDualMode
|
||||
{
|
||||
Eval,
|
||||
Derivative
|
||||
};
|
||||
|
||||
// Number of Active inputs and their argument indices, in ascending order.
|
||||
// A qfunction may have several Active inputs at once: e.g. a field's value
|
||||
// u and its gradient dudx both feed the output and both must be
|
||||
// differentiated (the chain-rule contraction with the value/gradient shape
|
||||
// functions then happens at the FE-operator level). We produce one gradient
|
||||
// block, d(output)/d(input), per Active input — each computed with the other
|
||||
// Active inputs frozen, so they come out as isolated partials.
|
||||
template <typename activity_t, size_t num_inputs, size_t... Is>
|
||||
constexpr size_t count_active_inputs(std::index_sequence<Is...>)
|
||||
{
|
||||
return ((Is < num_inputs && qf_param_is_active_v<activity_t, Is>
|
||||
? size_t{1} : size_t{0}) + ...);
|
||||
}
|
||||
|
||||
template <typename activity_t, size_t num_inputs, size_t num_active,
|
||||
size_t... Is>
|
||||
constexpr std::array<size_t, num_active>
|
||||
collect_active_inputs(std::index_sequence<Is...>)
|
||||
{
|
||||
std::array<size_t, num_active> idx{};
|
||||
size_t j = 0;
|
||||
(((Is < num_inputs && qf_param_is_active_v<activity_t, Is>)
|
||||
? (idx[j++] = Is) : size_t{0}), ...);
|
||||
return idx;
|
||||
}
|
||||
|
||||
template <typename Func, typename InputActivityTuple,
|
||||
typename OutputActivityTuple,
|
||||
RevDiffDualMode mode = RevDiffDualMode::Eval>
|
||||
struct RevDiff
|
||||
{
|
||||
using traits = function_traits<decltype(&Func::operator())>;
|
||||
using args_tuple = typename traits::args_tuple;
|
||||
using activity =
|
||||
decltype(concat_tuples(InputActivityTuple{}, OutputActivityTuple{}));
|
||||
static constexpr size_t arity = traits::arity;
|
||||
static constexpr size_t num_inputs = tuple_size<InputActivityTuple>::value;
|
||||
|
||||
static_assert(std::is_void_v<typename traits::primal_return_type>,
|
||||
"RevDiff only supports primal functions with void return type");
|
||||
static_assert(tuple_size<activity>::value == arity,
|
||||
"Number of input and output activity tags must match function "
|
||||
"arity");
|
||||
|
||||
static constexpr size_t num_active_inputs =
|
||||
count_active_inputs<activity, num_inputs>(
|
||||
std::make_index_sequence<arity> {});
|
||||
|
||||
static constexpr auto active_inputs =
|
||||
collect_active_inputs<activity, num_inputs, num_active_inputs>(
|
||||
std::make_index_sequence<arity> {});
|
||||
|
||||
// Slot index of argument I in the active_inputs array (compile-time).
|
||||
template <size_t I>
|
||||
static constexpr size_t slot_of()
|
||||
{
|
||||
for (size_t s = 0; s < num_active_inputs; s++)
|
||||
if (active_inputs[s] == I) { return s; }
|
||||
return num_active_inputs;
|
||||
}
|
||||
|
||||
static constexpr size_t active_output =
|
||||
find_single_active_qparam<activity, num_inputs, arity>();
|
||||
static_assert(active_output < arity,
|
||||
"gradient mode requires exactly one Active output");
|
||||
static_assert(num_active_inputs >= 1,
|
||||
"gradient mode requires at least one Active input");
|
||||
static_assert(tuple_size<OutputActivityTuple>::value == 1,
|
||||
"gradient mode requires exactly one (scalar) output");
|
||||
|
||||
using output_type =
|
||||
std::decay_t<tuple_element_t<active_output, args_tuple>>;
|
||||
|
||||
using output_view = typename qp_traits<output_type>::view_type;
|
||||
|
||||
static_assert(qp_traits<output_type>::components == 1,
|
||||
"gradient output requires a scalar output");
|
||||
|
||||
template <size_t I>
|
||||
using primal_arg_t = tuple_element_t<I, args_tuple>;
|
||||
|
||||
// True when reverse mode is served by the dual-number fallback rather than
|
||||
// Enzyme. Everything below that widens a type to a dual is gated on this, so
|
||||
// an Enzyme build sees exactly the types it saw before nested duals existed.
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
static constexpr bool native_dual_backend = false;
|
||||
#else
|
||||
static constexpr bool native_dual_backend = true;
|
||||
#endif
|
||||
|
||||
static constexpr bool use_native_dual_derivative =
|
||||
native_dual_backend && (mode == RevDiffDualMode::Derivative);
|
||||
|
||||
// Under the native-dual second-derivative path the active primals arrive
|
||||
// carrying the outer direction, so they must be dual-typed.
|
||||
template <size_t I>
|
||||
using derivative_arg_t =
|
||||
std::conditional_t<use_native_dual_derivative &&
|
||||
qf_param_is_active_v<activity, I>,
|
||||
typename qp_traits<std::decay_t<tuple_element_t<I, args_tuple>>>::dual_type,
|
||||
primal_arg_t<I>>;
|
||||
|
||||
template <size_t S>
|
||||
using active_arg_decay_t =
|
||||
std::decay_t<tuple_element_t<active_inputs[S], args_tuple>>;
|
||||
|
||||
template <size_t S>
|
||||
static constexpr bool active_arg_uses_dual()
|
||||
{
|
||||
return native_dual_backend &&
|
||||
(is_dual_number<active_arg_decay_t<S>>::value ||
|
||||
qf_param_uses_dual_v<active_arg_decay_t<S>>);
|
||||
}
|
||||
|
||||
// A gradient block mirrors its active input's shape. It needs a dual scalar
|
||||
// whenever the fallback has to return a value and a tangent through it.
|
||||
template <size_t S>
|
||||
using grad_arg_t =
|
||||
std::conditional_t<use_native_dual_derivative || active_arg_uses_dual<S>(),
|
||||
typename qp_traits<active_arg_decay_t<S>>::dual_type,
|
||||
typename qp_traits<active_arg_decay_t<S>>::view_type>
|
||||
&;
|
||||
|
||||
template <typename qfunc_type>
|
||||
using qfunc_args_tuple_t =
|
||||
decay_tuple<typename function_traits<decltype(&qfunc_type::operator())>::args_tuple>;
|
||||
|
||||
template <size_t... Is, size_t... Ss>
|
||||
static FunctionSignature<void(derivative_arg_t<Is>..., grad_arg_t<Ss>...)>
|
||||
signature_impl(std::index_sequence<Is...>, std::index_sequence<Ss...>);
|
||||
|
||||
using signature = decltype(signature_impl(std::make_index_sequence<num_inputs> {},
|
||||
std::make_index_sequence<num_active_inputs> {}));
|
||||
|
||||
Func func {};
|
||||
|
||||
RevDiff() = default;
|
||||
MFEM_HOST_DEVICE explicit RevDiff(const Func &func_) : func(func_) { }
|
||||
|
||||
// Plain function with the qfunction's exact (reference) signature, plus the
|
||||
// configured qfunction instance, so it can be handed to Enzyme as a function
|
||||
// pointer without default-constructing away runtime qfunction state.
|
||||
template <size_t... Is>
|
||||
MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void
|
||||
static_call(Func *func, tuple_element_t<Is, args_tuple>... args)
|
||||
{
|
||||
(*func)(args...);
|
||||
}
|
||||
|
||||
template <size_t... Is>
|
||||
static constexpr auto fn_ptr(std::index_sequence<Is...>)
|
||||
{
|
||||
return &static_call<Is...>;
|
||||
}
|
||||
static constexpr auto fn()
|
||||
{
|
||||
return fn_ptr(std::make_index_sequence<arity> {});
|
||||
}
|
||||
|
||||
// Load primal inputs from the pointer tuple into a local qargs copy. Dual
|
||||
// gradient parts are implicitly zero because qargs is value-initialized.
|
||||
template <typename QArgs, typename AllPtrs, size_t... Is>
|
||||
MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void load_qargs(
|
||||
QArgs &qargs, AllPtrs &ptrs, std::index_sequence<Is...>)
|
||||
{
|
||||
((mfem::future::get<int(Is)>(qargs) =
|
||||
*mfem::future::get<int(Is)>(ptrs)), ...);
|
||||
}
|
||||
|
||||
template <typename QArgs, typename AllPtrs, size_t... Is>
|
||||
MFEM_HOST_DEVICE static MFEM_FUTURE_ALWAYS_INLINE void
|
||||
lift_qargs_to_nested_dual(QArgs &qargs, AllPtrs &ptrs,
|
||||
std::index_sequence<Is...>)
|
||||
{
|
||||
(lift_to_nested_arg(*mfem::future::get<int(Is)>(ptrs),
|
||||
mfem::future::get<Is>(qargs)), ...);
|
||||
}
|
||||
|
||||
// The nested-dual q-function is a *different* type — its scalar template
|
||||
// parameter is rebound — so a configured instance cannot simply be copied
|
||||
// over. Runtime q-function state must still survive, or the second
|
||||
// derivative would silently be taken of a differently-parameterised energy.
|
||||
//
|
||||
// Three cases, in order:
|
||||
// * the rebound type converts from this one: use that conversion;
|
||||
// * no state at all: nothing to carry;
|
||||
// * same size and trivially copyable: none of the members depend on the
|
||||
// rebound scalar, so the two are layout-identical and the state copies
|
||||
// over bytewise. A member that *did* depend on the scalar would change
|
||||
// the size and land in the static_assert below instead.
|
||||
template <typename nested_func_t>
|
||||
MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE nested_func_t
|
||||
make_nested_func() const
|
||||
{
|
||||
if constexpr (std::is_constructible_v<nested_func_t, const Func &>)
|
||||
{
|
||||
return nested_func_t(func);
|
||||
}
|
||||
else if constexpr (std::is_empty_v<Func>)
|
||||
{
|
||||
return nested_func_t {};
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(std::is_trivially_copyable_v<Func> &&
|
||||
std::is_trivially_copyable_v<nested_func_t> &&
|
||||
sizeof(Func) == sizeof(nested_func_t),
|
||||
"second derivatives on the native dual backend rebind "
|
||||
"the q-function's scalar type; a q-function whose state "
|
||||
"depends on that scalar must be constructible from its "
|
||||
"rebound form");
|
||||
nested_func_t nested {};
|
||||
const auto *src = reinterpret_cast<const unsigned char *>(&func);
|
||||
auto *dst = reinterpret_cast<unsigned char *>(&nested);
|
||||
for (size_t b = 0; b < sizeof(Func); b++) { dst[b] = src[b]; }
|
||||
return nested;
|
||||
}
|
||||
}
|
||||
|
||||
// Seed the s-th Active input one component at a time and read the resulting
|
||||
// gradient block back out. This is the forward-mode dual-number stand-in for
|
||||
// one reverse-mode call: O(ncomp) evaluations instead of O(1).
|
||||
template <size_t S, typename AllPtrs>
|
||||
MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void seed_active_input(
|
||||
AllPtrs &ptrs) const
|
||||
{
|
||||
constexpr size_t input_idx = active_inputs[S];
|
||||
using active_arg_t = std::decay_t<tuple_element_t<input_idx, args_tuple>>;
|
||||
constexpr int ncomp = qp_traits<active_arg_t>::components;
|
||||
|
||||
for (int component = 0; component < ncomp; component++)
|
||||
{
|
||||
if constexpr (mode == RevDiffDualMode::Eval)
|
||||
{
|
||||
// Fresh value-initialized qargs: primals loaded below, all dual
|
||||
// gradient parts start at zero, so no explicit clear is needed.
|
||||
qfunc_args_tuple_t<Func> qargs {};
|
||||
load_qargs(qargs, ptrs, std::make_index_sequence<num_inputs> {});
|
||||
|
||||
auto &grad = *mfem::future::get<num_inputs + S>(ptrs);
|
||||
|
||||
qf_set_flat_gradient(mfem::future::get<input_idx>(qargs), component,
|
||||
1.0);
|
||||
|
||||
call_qfunc_no_move(func, qargs);
|
||||
|
||||
auto &out = mfem::future::get<active_output>(qargs);
|
||||
|
||||
qf_set_flat_value(grad, component, qf_flat_gradient(out, 0));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Lift the incoming dual (a, b) to ((a, c), (b, d)): b is the outer
|
||||
// Hessian-action direction, c is this loop's component seed. After
|
||||
// evaluating E the nested output holds ((E, dE/dx_i), (E'[b],
|
||||
// H_i[b])), and we hand dfem back (dE/dx_i, H_i[b]).
|
||||
static_assert(rebind_qfunc_scalar<Func, nested_native_dual_t>::supported,
|
||||
"RevDiff native-dual derivative mode requires "
|
||||
"q-function types of the form QFunc<scalar_t> so they "
|
||||
"can be rebound to nested dual scalars");
|
||||
using nested_func_t = rebind_qfunc_scalar_t<Func, nested_native_dual_t>;
|
||||
qfunc_args_tuple_t<nested_func_t> nested_qargs {};
|
||||
|
||||
lift_qargs_to_nested_dual(nested_qargs, ptrs,
|
||||
std::make_index_sequence<num_inputs> {});
|
||||
|
||||
qf_set_flat_value_gradient(
|
||||
mfem::future::get<input_idx>(nested_qargs), component, 1.0);
|
||||
|
||||
call_qfunc_no_move(make_nested_func<nested_func_t>(), nested_qargs);
|
||||
|
||||
auto &out = mfem::future::get<active_output>(nested_qargs);
|
||||
auto &grad = *mfem::future::get<num_inputs + S>(ptrs);
|
||||
|
||||
qf_set_flat_value(grad, component, qf_flat_value_gradient(out, 0));
|
||||
qf_set_flat_gradient(grad, component,
|
||||
qf_flat_gradient_gradient(out, 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Dual-number fallback for the whole reverse-mode call: one seeded sweep per
|
||||
// Active input.
|
||||
template <typename AllPtrs>
|
||||
MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void call_dual_rev(
|
||||
AllPtrs &ptrs) const
|
||||
{
|
||||
for_constexpr<num_active_inputs>([&](auto s)
|
||||
{
|
||||
seed_active_input<decltype(s)::value>(ptrs);
|
||||
});
|
||||
}
|
||||
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
// Recursive builder of the per-argument reverse-mode enzyme call.
|
||||
template <size_t I = 0, typename AllPtrs, typename... Built>
|
||||
MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void
|
||||
call_enzyme_rev(AllPtrs &ptrs, output_view &scratch, output_view &adjoint,
|
||||
Built... built) const
|
||||
{
|
||||
if constexpr (I == arity)
|
||||
{
|
||||
__enzyme_autodiff<void>(fn(), enzyme_const, const_cast<Func *>(&func),
|
||||
built...);
|
||||
}
|
||||
else if constexpr (I == active_output)
|
||||
{
|
||||
// Output: primal written to scratch (unused), adjoint seeded to 1.
|
||||
call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
|
||||
enzyme_dupnoneed, &scratch, &adjoint);
|
||||
}
|
||||
else if constexpr (qf_param_is_active_v<activity, I>)
|
||||
{
|
||||
// Active input: gradient accumulates into its grad-output slot.
|
||||
call_enzyme_rev<I + 1>(
|
||||
ptrs, scratch, adjoint, built..., enzyme_dup,
|
||||
mfem::future::get<int(I)>(ptrs),
|
||||
mfem::future::get<int(num_inputs + slot_of<I>())>(ptrs));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Const input: primal only, no shadow.
|
||||
call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
|
||||
enzyme_const, mfem::future::get<int(I)>(ptrs));
|
||||
}
|
||||
}
|
||||
#endif // MFEM_USE_ENZYME
|
||||
|
||||
// Zero all gradient outputs before the enzyme call (Enzyme accumulates).
|
||||
template <typename AllPtrs, size_t... Ss>
|
||||
MFEM_HOST_DEVICE static
|
||||
MFEM_FUTURE_ALWAYS_INLINE void zero_grads(
|
||||
AllPtrs &ptrs,
|
||||
std::index_sequence<Ss...>)
|
||||
{
|
||||
((*mfem::future::get<int(num_inputs + Ss)>(ptrs) =
|
||||
std::decay_t<decltype(*mfem::future::get<int(num_inputs + Ss)>(ptrs))> {}),
|
||||
...);
|
||||
}
|
||||
|
||||
// Called once per quadrature point. Arguments are, in order:
|
||||
// * the primal value of every qfunction input (active and const), then
|
||||
// * one gradient output per Active input (ascending index order), each
|
||||
// shaped like its Active input.
|
||||
// The qfunction's own output slot is not passed; Enzyme writes it to stack
|
||||
// scratch (enzyme_dupnoneed). A single __enzyme_autodiff call yields all
|
||||
// gradient blocks simultaneously.
|
||||
template <typename... Args>
|
||||
MFEM_HOST_DEVICE MFEM_FUTURE_ALWAYS_INLINE void operator()(
|
||||
Args &&...args) const
|
||||
{
|
||||
static_assert(sizeof...(Args) == num_inputs + num_active_inputs,
|
||||
"expected one primal per input plus one gradient output per "
|
||||
"Active input");
|
||||
auto ptrs = mfem::future::make_tuple(&args...);
|
||||
zero_grads(ptrs, std::make_index_sequence<num_active_inputs> {});
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
output_view out_scratch {};
|
||||
output_view out_adjoint{1.0}; // seed: d(output)/d(output) = 1
|
||||
call_enzyme_rev(ptrs, out_scratch, out_adjoint);
|
||||
#else
|
||||
call_dual_rev(ptrs);
|
||||
#endif
|
||||
}
|
||||
|
||||
static MFEM_FUTURE_ALWAYS_INLINE void print() { print_impl(std::make_index_sequence<arity> {}); }
|
||||
|
||||
template <size_t... Is> static MFEM_FUTURE_ALWAYS_INLINE void print_impl(
|
||||
std::index_sequence<Is...>)
|
||||
{
|
||||
mfem::out << "__enzyme_autodiff<void>(fptr";
|
||||
(([&]
|
||||
{
|
||||
auto name = get_type_name<tuple_element_t<Is, args_tuple>>();
|
||||
if constexpr (Is == active_output)
|
||||
mfem::out << ", enzyme_dupnoneed, " << name << ", adjoint=1";
|
||||
else if constexpr (qf_param_is_active_v<activity, Is>)
|
||||
mfem::out << ", enzyme_dup, " << name << ", grad out";
|
||||
else
|
||||
{
|
||||
mfem::out << ", enzyme_const, " << name;
|
||||
}
|
||||
}()),
|
||||
...);
|
||||
mfem::out << ")\n";
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Func, typename InputActivityTuple,
|
||||
typename OutputActivityTuple, RevDiffDualMode mode>
|
||||
struct create_function_signature<RevDiff<Func, InputActivityTuple,
|
||||
OutputActivityTuple, mode>>
|
||||
{
|
||||
using type = typename
|
||||
RevDiff<Func, InputActivityTuple, OutputActivityTuple, mode>::signature;
|
||||
};
|
||||
|
||||
/// Builds the reverse-mode transform of @a f, differentiating the inputs marked
|
||||
/// Active in @a activity_t.
|
||||
///
|
||||
/// A factory rather than a plain declaration of a RevDiff variable for compatibility with MSVC.
|
||||
template <typename activity_t, RevDiffDualMode mode = RevDiffDualMode::Eval,
|
||||
typename func_t>
|
||||
auto make_revdiff(const func_t &f)
|
||||
{
|
||||
return RevDiff<func_t, activity_t, tuple<Active>, mode>(f);
|
||||
}
|
||||
|
||||
} // namespace mfem::future
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,327 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
#include "../../../general/error.hpp"
|
||||
#include "../../../linalg/vector.hpp"
|
||||
#include "../tuple.hpp"
|
||||
#include <initializer_list>
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
// Scratch storage and q-function shadow helpers for dFEM backends. The bank
|
||||
// supports two scratch kinds:
|
||||
// - quadrature-point scratch: real_t buffers sized as NQ * components_per_qp,
|
||||
// - global scratch: one tuple of qfunction-local temporaries, independent of
|
||||
// NQ, used for values such as flags, scalars, or small Vector workspaces.
|
||||
//
|
||||
// @a scalar_t is the scalar the owning q-function uses at a quadrature point.
|
||||
// With Enzyme this is real_t and the tangent lives in a separate shadow bank.
|
||||
// Without Enzyme the q-function is evaluated on native duals, which carry the
|
||||
// tangent inside the value itself; the bank then widens its backing storage
|
||||
// accordingly so that a scratch entry can round-trip a dual without dropping
|
||||
// the gradient. Backing storage stays a real_t Vector in both cases, so the
|
||||
// device and shadow plumbing is unchanged.
|
||||
template <typename scalar_t, typename... GlobalScratchTypes>
|
||||
struct ScratchBank
|
||||
{
|
||||
static_assert(sizeof(scalar_t) % sizeof(real_t) == 0,
|
||||
"scratch scalar must be a whole number of real_t");
|
||||
|
||||
/// Number of real_t needed to back one scalar_t scratch entry.
|
||||
static constexpr int scalar_size = sizeof(scalar_t) / sizeof(real_t);
|
||||
|
||||
|
||||
//=================================
|
||||
///<--- Global scratch utilities.
|
||||
//=================================
|
||||
|
||||
using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
|
||||
|
||||
template <typename T>
|
||||
static T MakeGlobalScratchShadow(const T &)
|
||||
{
|
||||
return T {};
|
||||
}
|
||||
|
||||
static Vector MakeGlobalScratchShadow(const Vector &primal)
|
||||
{
|
||||
Vector shadow(primal.Size());
|
||||
shadow.UseDevice(true);
|
||||
shadow = 0.0;
|
||||
return shadow;
|
||||
}
|
||||
|
||||
template <typename Tuple, size_t... Is>
|
||||
static auto MakeGlobalScratchShadowTuple(const Tuple &primal,
|
||||
std::index_sequence<Is...>)
|
||||
{
|
||||
return make_tuple(MakeGlobalScratchShadow(get<Is>(primal))...);
|
||||
}
|
||||
|
||||
template <typename Tuple>
|
||||
static auto MakeGlobalScratchShadowTuple(const Tuple &primal)
|
||||
{
|
||||
return MakeGlobalScratchShadowTuple(
|
||||
primal, std::make_index_sequence<tuple_size<Tuple>::value> {});
|
||||
}
|
||||
|
||||
|
||||
//===========================
|
||||
///<--- Scratch objects
|
||||
//===========================
|
||||
|
||||
mutable GlobalScratchTuple global;
|
||||
|
||||
int nq = 0;
|
||||
std::vector<int> components;
|
||||
std::vector<int> sizes;
|
||||
std::vector<std::shared_ptr<Vector>> owned;
|
||||
std::vector<real_t *> ptrs;
|
||||
|
||||
|
||||
//===========================
|
||||
///<--- Setter methods
|
||||
//===========================
|
||||
|
||||
void SetScratch(const int nq_,
|
||||
std::initializer_list<int> components_per_qp = {1})
|
||||
{
|
||||
SetScratch(nq_, std::vector<int>(components_per_qp));
|
||||
}
|
||||
|
||||
void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
|
||||
{
|
||||
nq = nq_;
|
||||
components.clear();
|
||||
sizes.clear();
|
||||
owned.clear();
|
||||
ptrs.clear();
|
||||
for (int component_count : components_per_qp)
|
||||
{
|
||||
AddScratch(component_count);
|
||||
}
|
||||
}
|
||||
|
||||
void AddScratch(const int components_per_qp = 1)
|
||||
{
|
||||
MFEM_VERIFY(nq > 0, "SetScratch must be called before AddScratch");
|
||||
MFEM_VERIFY(components_per_qp > 0,
|
||||
"scratch components per quadrature point must be positive");
|
||||
owned.push_back(std::make_shared<Vector>());
|
||||
Vector &scratch = *owned.back();
|
||||
const int size = components_per_qp * nq * scalar_size;
|
||||
scratch.SetSize(size);
|
||||
scratch.UseDevice(true);
|
||||
scratch = 0.0;
|
||||
components.push_back(components_per_qp);
|
||||
sizes.push_back(scratch.Size());
|
||||
ptrs.push_back(scratch.ReadWrite());
|
||||
}
|
||||
|
||||
void SetGlobalScratch(const GlobalScratchTuple &global_)
|
||||
{
|
||||
global = global_;
|
||||
}
|
||||
|
||||
|
||||
//===========================
|
||||
///<--- Getter methods
|
||||
//===========================
|
||||
|
||||
/// Scratch buffer @a i viewed as the q-function's scalar type.
|
||||
scalar_t *GetScratchPointer(const int i) const
|
||||
{
|
||||
return reinterpret_cast<scalar_t *>(ptrs[i]);
|
||||
}
|
||||
|
||||
scalar_t *operator[](const int i) const { return GetScratchPointer(i); }
|
||||
|
||||
/// Raw real_t backing storage of scratch buffer @a i. Its size is
|
||||
/// scalar_size times the number of scalar_t entries.
|
||||
Vector &GetScratchVector(const int i) const { return *owned[i]; }
|
||||
|
||||
template <int I>
|
||||
auto &GetGlobalScratch() const
|
||||
{
|
||||
return get<I>(global);
|
||||
}
|
||||
|
||||
|
||||
//===========================
|
||||
///<--- Utils methods
|
||||
//===========================
|
||||
|
||||
void CloneScratchLayoutTo(ScratchBank &shadow) const
|
||||
{
|
||||
shadow.SetScratch(nq, components);
|
||||
shadow.SetGlobalScratch(MakeGlobalScratchShadowTuple(global));
|
||||
}
|
||||
|
||||
int Size() const { return static_cast<int>(ptrs.size()); }
|
||||
};
|
||||
|
||||
// Shared base for Q-functions that use ScratchBank. Under Enzyme a matching
|
||||
// scratch shadow is created for forward differentiation; with native duals the
|
||||
// tangent rides along in the scratch entry and no shadow is created.
|
||||
template <typename scalar_t, typename... GlobalScratchTypes>
|
||||
struct QFWithScratch
|
||||
{
|
||||
using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
|
||||
using ScratchScalar = scalar_t;
|
||||
|
||||
/// Number of real_t backing one scratch entry; see ScratchBank.
|
||||
static constexpr int scalar_size =
|
||||
ScratchBank<scalar_t, GlobalScratchTypes...>::scalar_size;
|
||||
|
||||
int nq = 0;
|
||||
ScratchBank<scalar_t, GlobalScratchTypes...> scratch;
|
||||
|
||||
void SetScratch(const int nq_,
|
||||
std::initializer_list<int> components_per_qp = {1})
|
||||
{
|
||||
nq = nq_;
|
||||
scratch.SetScratch(nq, components_per_qp);
|
||||
}
|
||||
|
||||
void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
|
||||
{
|
||||
nq = nq_;
|
||||
scratch.SetScratch(nq, components_per_qp);
|
||||
}
|
||||
|
||||
void SetScratch(const int nq_, const int num_scratch_elem,
|
||||
const int components_per_qp = 1)
|
||||
{
|
||||
nq = nq_;
|
||||
scratch.SetScratch(nq,
|
||||
std::vector<int>(num_scratch_elem, components_per_qp));
|
||||
}
|
||||
|
||||
void SetGlobalScratch(const GlobalScratchTuple &global_scratch_)
|
||||
{
|
||||
scratch.SetGlobalScratch(global_scratch_);
|
||||
}
|
||||
|
||||
Vector &GetScratchVector(const int i) const
|
||||
{
|
||||
return scratch.GetScratchVector(i);
|
||||
}
|
||||
|
||||
scalar_t *GetScratchPointer(const int i) const
|
||||
{
|
||||
return scratch.GetScratchPointer(i);
|
||||
}
|
||||
|
||||
template <int I>
|
||||
auto &GetGlobalScratch() const
|
||||
{
|
||||
return scratch.template GetGlobalScratch<I>();
|
||||
}
|
||||
|
||||
void CloneScratchLayoutTo(QFWithScratch &shadow) const
|
||||
{
|
||||
shadow.nq = nq;
|
||||
scratch.CloneScratchLayoutTo(shadow.scratch);
|
||||
}
|
||||
|
||||
QFWithScratch CreateShadow() const
|
||||
{
|
||||
QFWithScratch shadow;
|
||||
CloneScratchLayoutTo(shadow);
|
||||
return shadow;
|
||||
}
|
||||
};
|
||||
|
||||
/// Q-function base with quadrature-point scratch only. @a scalar_t is the
|
||||
/// scalar the q-function signature uses (real_t under Enzyme, dual otherwise).
|
||||
template <typename scalar_t = real_t>
|
||||
using QFWithScratchType = QFWithScratch<scalar_t>;
|
||||
|
||||
/// Q-function base with quadrature-point scratch and a global scratch tuple.
|
||||
template <typename scalar_t = real_t>
|
||||
using QFWithGlobalScratchType =
|
||||
QFWithScratch<scalar_t, bool, real_t, Vector>;
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
struct qfunc_uses_scratch
|
||||
{
|
||||
private:
|
||||
template <typename scalar_t, typename... GlobalScratchTypes>
|
||||
static std::true_type Test(
|
||||
const QFWithScratch<scalar_t, GlobalScratchTypes...> *);
|
||||
|
||||
static std::false_type Test(...);
|
||||
|
||||
public:
|
||||
static constexpr bool value = decltype(Test(
|
||||
static_cast<std::remove_cv_t<std::remove_reference_t<T>> *>(nullptr)))::value;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool qfunc_uses_scratch_v =
|
||||
qfunc_uses_scratch<T>::value;
|
||||
|
||||
struct unused_qfunc_shadow { };
|
||||
|
||||
// A separate shadow scratch bank only exists for Enzyme, which writes tangents
|
||||
// into shadow memory. The native dual fallback carries the tangent inside the
|
||||
// scratch entry itself (see ScratchBank::scalar_size), so a shadow bank would
|
||||
// be allocated and never read; it is dropped entirely there.
|
||||
template <typename T>
|
||||
inline constexpr bool qfunc_needs_shadow_v =
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
qfunc_uses_scratch_v<T>;
|
||||
#else
|
||||
false;
|
||||
#endif
|
||||
|
||||
template <typename qfunc_t, bool needs_shadow>
|
||||
struct qfunc_shadow_type
|
||||
{
|
||||
using type = unused_qfunc_shadow;
|
||||
};
|
||||
|
||||
template <typename qfunc_t>
|
||||
struct qfunc_shadow_type<qfunc_t, true>
|
||||
{
|
||||
using type = decltype(std::declval<const qfunc_t &>().CreateShadow());
|
||||
};
|
||||
|
||||
template <typename qfunc_t>
|
||||
using qfunc_shadow_t = typename qfunc_shadow_type<qfunc_t,
|
||||
qfunc_needs_shadow_v<qfunc_t>>::type;
|
||||
|
||||
// Create a persistent q-function shadow if one is needed, otherwise return an empty struct.
|
||||
template <typename qfunc_t>
|
||||
inline qfunc_shadow_t<qfunc_t> MakeQFunctionShadowStorage(
|
||||
const qfunc_t &qfunc)
|
||||
{
|
||||
if constexpr (qfunc_needs_shadow_v<qfunc_t>)
|
||||
{
|
||||
return qfunc.CreateShadow();
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_CONTRACT_VAR(qfunc);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+19
-341
@@ -11,366 +11,44 @@
|
||||
|
||||
#include "doperator.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#ifdef MFEM_USE_MPI
|
||||
|
||||
using namespace mfem;
|
||||
using namespace mfem::future;
|
||||
|
||||
namespace
|
||||
void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
|
||||
{
|
||||
int GetTotalTrueVSize(const std::vector<FieldDescriptor> &fds)
|
||||
{
|
||||
int size = 0;
|
||||
for (const auto &fd : fds)
|
||||
MFEM_ASSERT(parameters.size() == p.size(),
|
||||
"number of parameters doesn't match descriptors");
|
||||
for (size_t i = 0; i < parameters.size(); i++)
|
||||
{
|
||||
size += mfem::future::GetTrueVSize(fd);
|
||||
p[i]->Read();
|
||||
parameters_l[i] = *p[i];
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
template <typename map_t>
|
||||
const typename map_t::mapped_type &FindOrDefault(
|
||||
const map_t &map, const typename map_t::key_type &id,
|
||||
const typename map_t::mapped_type &fallback)
|
||||
{
|
||||
const auto it = map.find(id);
|
||||
return it == map.end() ? fallback : it->second;
|
||||
}
|
||||
|
||||
template <typename map_t>
|
||||
typename map_t::mapped_type FindOrEmpty(
|
||||
const map_t &map, const typename map_t::key_type &id)
|
||||
{
|
||||
const auto it = map.find(id);
|
||||
return it == map.end() ? typename map_t::mapped_type{} : it->second;
|
||||
}
|
||||
|
||||
const std::vector<derivative_action_t> &SelectActionCallbacks(
|
||||
const std::vector<derivative_action_t> &direct_actions,
|
||||
const DerivativeActionMap &cached_actions,
|
||||
size_t derivative_id,
|
||||
bool use_cached_setup)
|
||||
{
|
||||
if (use_cached_setup)
|
||||
{
|
||||
const auto it_apply = cached_actions.find(derivative_id);
|
||||
if (it_apply != cached_actions.end() && !it_apply->second.empty())
|
||||
{
|
||||
return it_apply->second;
|
||||
}
|
||||
}
|
||||
|
||||
return direct_actions;
|
||||
}
|
||||
|
||||
struct DerivativeCallbackSet
|
||||
{
|
||||
const DerivativeActionMap &actions;
|
||||
const DerivativeActionMap &cached_actions;
|
||||
const DerivativeActionMap &transpose_actions;
|
||||
const DerivativeFieldMap &outfds;
|
||||
const SparseAssemblyMap &assemble_sparse;
|
||||
const HypreAssemblyMap &assemble_hypre;
|
||||
const DiagonalAssemblyMap &assemble_diagonal;
|
||||
const DerivativeSetupMap &setup;
|
||||
const char *missing_action_message;
|
||||
};
|
||||
|
||||
struct SecondDerivativeCallbackSet
|
||||
{
|
||||
const SecondDerivativeActionMap &actions;
|
||||
const SecondDerivativeActionMap &cached_actions;
|
||||
const SecondDerivativeActionMap &transpose_actions;
|
||||
const SecondDerivativeFieldMap &outfds;
|
||||
const SecondSparseAssemblyMap &assemble_sparse;
|
||||
const SecondHypreAssemblyMap &assemble_hypre;
|
||||
const SecondDiagonalAssemblyMap &assemble_diagonal;
|
||||
const SecondDerivativeSetupMap &setup;
|
||||
const char *missing_action_message;
|
||||
};
|
||||
|
||||
template <typename vector_t>
|
||||
std::shared_ptr<DerivativeOperator> MakeStatefulDerivativeOperator(
|
||||
size_t derivative_id,
|
||||
const vector_t &x,
|
||||
const std::vector<FieldDescriptor> &infds,
|
||||
const std::vector<FieldDescriptor> &default_outfds,
|
||||
const DerivativeCallbackSet &callbacks,
|
||||
bool use_cached_setup,
|
||||
bool lvector_mode,
|
||||
bool functional_gradient = false)
|
||||
{
|
||||
const auto it_action = callbacks.actions.find(derivative_id);
|
||||
MFEM_ASSERT(it_action != callbacks.actions.end(),
|
||||
callbacks.missing_action_message << derivative_id);
|
||||
|
||||
const size_t dfidx = FindIdx(derivative_id, infds);
|
||||
const auto &doutfds =
|
||||
FindOrDefault(callbacks.outfds, derivative_id, default_outfds);
|
||||
const auto &mult_callbacks =
|
||||
SelectActionCallbacks(it_action->second, callbacks.cached_actions,
|
||||
derivative_id, use_cached_setup);
|
||||
|
||||
return std::make_shared<DerivativeOperator>(
|
||||
GetTotalTrueVSize(doutfds),
|
||||
GetTrueVSize(infds[dfidx]),
|
||||
mult_callbacks,
|
||||
FindOrEmpty(callbacks.transpose_actions, derivative_id),
|
||||
infds[dfidx],
|
||||
x,
|
||||
infds,
|
||||
doutfds,
|
||||
FindOrEmpty(callbacks.assemble_sparse, derivative_id),
|
||||
FindOrEmpty(callbacks.assemble_hypre, derivative_id),
|
||||
FindOrEmpty(callbacks.assemble_diagonal, derivative_id),
|
||||
FindOrEmpty(callbacks.setup, derivative_id),
|
||||
lvector_mode,
|
||||
functional_gradient);
|
||||
}
|
||||
|
||||
const std::vector<derivative_action_t> &SelectSecondDerivativeActionCallbacks(
|
||||
const std::vector<derivative_action_t> &direct_actions,
|
||||
const SecondDerivativeActionMap &cached_actions,
|
||||
second_derivative_key_t derivative_key,
|
||||
bool use_cached_setup)
|
||||
{
|
||||
if (use_cached_setup)
|
||||
{
|
||||
const auto it_apply = cached_actions.find(derivative_key);
|
||||
if (it_apply != cached_actions.end() && !it_apply->second.empty())
|
||||
{
|
||||
return it_apply->second;
|
||||
}
|
||||
}
|
||||
|
||||
return direct_actions;
|
||||
}
|
||||
|
||||
template <typename vector_t>
|
||||
std::shared_ptr<DerivativeOperator> MakeStatefulSecondDerivativeOperator(
|
||||
size_t gradient_id,
|
||||
size_t direction_id,
|
||||
const vector_t &x,
|
||||
const std::vector<FieldDescriptor> &infds,
|
||||
const std::vector<FieldDescriptor> &default_outfds,
|
||||
const SecondDerivativeCallbackSet &callbacks,
|
||||
bool use_cached_setup,
|
||||
bool lvector_mode)
|
||||
{
|
||||
const second_derivative_key_t derivative_key{gradient_id, direction_id};
|
||||
const auto it_action = callbacks.actions.find(derivative_key);
|
||||
MFEM_ASSERT(it_action != callbacks.actions.end(),
|
||||
callbacks.missing_action_message << "(" << gradient_id << ", "
|
||||
<< direction_id << ")");
|
||||
|
||||
const size_t dfidx = FindIdx(direction_id, infds);
|
||||
const auto &doutfds =
|
||||
FindOrDefault(callbacks.outfds, derivative_key, default_outfds);
|
||||
const auto &mult_callbacks =
|
||||
SelectSecondDerivativeActionCallbacks(
|
||||
it_action->second, callbacks.cached_actions, derivative_key,
|
||||
use_cached_setup);
|
||||
|
||||
return std::make_shared<DerivativeOperator>(
|
||||
GetTotalTrueVSize(doutfds),
|
||||
GetTrueVSize(infds[dfidx]),
|
||||
mult_callbacks,
|
||||
FindOrEmpty(callbacks.transpose_actions, derivative_key),
|
||||
infds[dfidx],
|
||||
x,
|
||||
infds,
|
||||
doutfds,
|
||||
FindOrEmpty(callbacks.assemble_sparse, derivative_key),
|
||||
FindOrEmpty(callbacks.assemble_hypre, derivative_key),
|
||||
FindOrEmpty(callbacks.assemble_diagonal, derivative_key),
|
||||
FindOrEmpty(callbacks.setup, derivative_key),
|
||||
lvector_mode);
|
||||
}
|
||||
}
|
||||
|
||||
DifferentiableOperator::DifferentiableOperator(
|
||||
const std::vector<FieldDescriptor> &infds,
|
||||
const std::vector<FieldDescriptor> &outfds,
|
||||
const std::vector<FieldDescriptor> &solutions,
|
||||
const std::vector<FieldDescriptor> ¶meters,
|
||||
const ParMesh &mesh) :
|
||||
Operator(),
|
||||
mesh(mesh),
|
||||
infds(infds),
|
||||
outfds(outfds)
|
||||
solutions(solutions),
|
||||
parameters(parameters)
|
||||
{
|
||||
unionfds.clear();
|
||||
unionfds.insert(unionfds.end(), infds.begin(), infds.end());
|
||||
unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
|
||||
std::sort(unionfds.begin(), unionfds.end());
|
||||
auto last = std::unique(unionfds.begin(), unionfds.end());
|
||||
unionfds.erase(last, unionfds.end());
|
||||
fields.resize(solutions.size() + parameters.size());
|
||||
fields_e.resize(fields.size());
|
||||
solutions_l.resize(solutions.size());
|
||||
parameters_l.resize(parameters.size());
|
||||
|
||||
infields_l.resize(infds.size());
|
||||
for (size_t i = 0; i < infds.size(); i++)
|
||||
for (size_t i = 0; i < solutions.size(); i++)
|
||||
{
|
||||
infields_l[i] = new Vector(GetVSize(infds[i]));
|
||||
fields[i] = solutions[i];
|
||||
}
|
||||
|
||||
infields_e.resize(infds.size());
|
||||
}
|
||||
|
||||
void DifferentiableOperator::SetMultLevel(MultLevel level)
|
||||
{
|
||||
mult_level = level;
|
||||
}
|
||||
|
||||
void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
|
||||
{
|
||||
MFEM_ASSERT(!action_callbacks.empty(),
|
||||
"no integrators have been set");
|
||||
|
||||
MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
|
||||
"x needs to be a BlockVector");
|
||||
|
||||
MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
|
||||
"y needs to be a BlockVector");
|
||||
|
||||
const auto &bx = static_cast<const BlockVector &>(x);
|
||||
auto &by = static_cast<BlockVector &>(y);
|
||||
|
||||
Mult(bx, by);
|
||||
}
|
||||
|
||||
void DifferentiableOperator::DisableTensorProductStructure(bool disable)
|
||||
{
|
||||
use_tensor_product_structure = !disable;
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
|
||||
size_t derivative_id, const Vector &x)
|
||||
{
|
||||
return MakeStatefulDerivativeOperator(
|
||||
derivative_id, x, infds, outfds,
|
||||
for (size_t i = 0; i < parameters.size(); i++)
|
||||
{
|
||||
derivative_action_callbacks,
|
||||
derivative_apply_callbacks,
|
||||
daction_transpose_callbacks,
|
||||
derivative_outfds,
|
||||
assemble_derivative_sparsematrix_callbacks,
|
||||
assemble_derivative_hypreparmatrix_callbacks,
|
||||
assemble_diagonal_callbacks,
|
||||
derivative_setup_callbacks,
|
||||
"no derivative action has been found for ID "
|
||||
},
|
||||
true,
|
||||
mult_level == MultLevel::LVECTOR,
|
||||
IsFunctionalDerivative(derivative_id));
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
|
||||
size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
|
||||
{
|
||||
return MakeStatefulDerivativeOperator(
|
||||
derivative_id, x, infds, outfds,
|
||||
{
|
||||
derivative_action_callbacks,
|
||||
derivative_apply_callbacks,
|
||||
daction_transpose_callbacks,
|
||||
derivative_outfds,
|
||||
assemble_derivative_sparsematrix_callbacks,
|
||||
assemble_derivative_hypreparmatrix_callbacks,
|
||||
assemble_diagonal_callbacks,
|
||||
derivative_setup_callbacks,
|
||||
"no derivative action has been found for ID "
|
||||
},
|
||||
use_cached_setup,
|
||||
mult_level == MultLevel::LVECTOR,
|
||||
IsFunctionalDerivative(derivative_id));
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
|
||||
size_t derivative_id)
|
||||
{
|
||||
MFEM_ASSERT(HasFunctionalIntegrator(),
|
||||
"stateless GetDerivative is available only for functionals");
|
||||
|
||||
const auto it_action = derivative_action_callbacks.find(derivative_id);
|
||||
MFEM_ASSERT(it_action != derivative_action_callbacks.end(),
|
||||
"no derivative action has been found for ID " << derivative_id);
|
||||
|
||||
const size_t dfidx = FindIdx(derivative_id, infds);
|
||||
const auto &doutfds =
|
||||
FindOrDefault(derivative_outfds, derivative_id, outfds);
|
||||
|
||||
return std::make_shared<DerivativeOperator>(
|
||||
GetTotalTrueVSize(doutfds),
|
||||
GetTrueVSize(infds[dfidx]),
|
||||
it_action->second,
|
||||
infds,
|
||||
doutfds);
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
|
||||
size_t derivative_id, const Vector &x)
|
||||
{
|
||||
return GetSecondDerivative(derivative_id, derivative_id, x);
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
|
||||
size_t gradient_id, size_t direction_id, const Vector &x)
|
||||
{
|
||||
MFEM_ASSERT(HasFunctionalIntegrator(),
|
||||
"second derivatives are available only for functionals");
|
||||
|
||||
return MakeStatefulSecondDerivativeOperator(
|
||||
gradient_id, direction_id, x, infds, outfds,
|
||||
{
|
||||
second_derivative_action_callbacks,
|
||||
second_derivative_apply_callbacks,
|
||||
second_daction_transpose_callbacks,
|
||||
second_derivative_outfds,
|
||||
assemble_second_derivative_sparsematrix_callbacks,
|
||||
assemble_second_derivative_hypreparmatrix_callbacks,
|
||||
assemble_second_derivative_diagonal_callbacks,
|
||||
second_derivative_setup_callbacks,
|
||||
"no second derivative action has been found for ID "
|
||||
},
|
||||
false,
|
||||
mult_level == MultLevel::LVECTOR);
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
|
||||
size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
|
||||
{
|
||||
return GetSecondDerivative(derivative_id, derivative_id, x, use_cached_setup);
|
||||
}
|
||||
|
||||
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
|
||||
size_t gradient_id, size_t direction_id, const MultiVector &x,
|
||||
const bool use_cached_setup)
|
||||
{
|
||||
MFEM_ASSERT(HasFunctionalIntegrator(),
|
||||
"second derivatives are available only for functionals");
|
||||
|
||||
return MakeStatefulSecondDerivativeOperator(
|
||||
gradient_id, direction_id, x, infds, outfds,
|
||||
{
|
||||
second_derivative_action_callbacks,
|
||||
second_derivative_apply_callbacks,
|
||||
second_daction_transpose_callbacks,
|
||||
second_derivative_outfds,
|
||||
assemble_second_derivative_sparsematrix_callbacks,
|
||||
assemble_second_derivative_hypreparmatrix_callbacks,
|
||||
assemble_second_derivative_diagonal_callbacks,
|
||||
second_derivative_setup_callbacks,
|
||||
"no second derivative action has been found for ID "
|
||||
},
|
||||
use_cached_setup,
|
||||
mult_level == MultLevel::LVECTOR);
|
||||
}
|
||||
|
||||
bool DifferentiableOperator::HasSecondDerivative(size_t gradient_id,
|
||||
size_t direction_id) const
|
||||
{
|
||||
const second_derivative_key_t derivative_key{gradient_id, direction_id};
|
||||
const auto it = second_derivative_action_callbacks.find(derivative_key);
|
||||
return it != second_derivative_action_callbacks.end() && !it->second.empty();
|
||||
fields[i + solutions.size()] = parameters[i];
|
||||
}
|
||||
}
|
||||
|
||||
#endif // MFEM_USE_MPI
|
||||
|
||||
+979
-1236
File diff suppressed because it is too large
Load Diff
@@ -1,64 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "../fespace.hpp"
|
||||
#include "../qspace.hpp"
|
||||
#include "parameterspace.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
/// @brief FieldDescriptor struct
|
||||
///
|
||||
/// This struct is used to store information about a field.
|
||||
struct FieldDescriptor
|
||||
{
|
||||
using data_variant_t =
|
||||
std::variant<const FiniteElementSpace *,
|
||||
const ParFiniteElementSpace *,
|
||||
const VectorQuadratureSpace *,
|
||||
const ParameterSpace *>;
|
||||
|
||||
/// Field ID
|
||||
std::size_t id;
|
||||
|
||||
/// Field variant
|
||||
data_variant_t data;
|
||||
|
||||
/// Default constructor
|
||||
FieldDescriptor() :
|
||||
id(SIZE_MAX), data(data_variant_t{}) {}
|
||||
|
||||
/// Constructor
|
||||
template <typename T>
|
||||
FieldDescriptor(std::size_t field_id, const T* v) :
|
||||
id(field_id), data(v) {}
|
||||
|
||||
bool operator==(const FieldDescriptor& other) const
|
||||
{
|
||||
return id == other.id;
|
||||
}
|
||||
|
||||
bool operator<(const FieldDescriptor& other) const
|
||||
{
|
||||
return id < other.id;
|
||||
}
|
||||
|
||||
friend void swap(FieldDescriptor& a, FieldDescriptor& b)
|
||||
{
|
||||
using std::swap;
|
||||
swap(a.id, b.id);
|
||||
swap(a.data, b.data);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@@ -10,7 +10,6 @@
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include <ostream>
|
||||
#include <type_traits>
|
||||
|
||||
namespace mfem::future
|
||||
@@ -75,15 +74,6 @@ struct is_identity_fop : std::false_type {};
|
||||
template <int FIELD_ID>
|
||||
struct is_identity_fop<Identity<FIELD_ID>> : std::true_type {};
|
||||
|
||||
template <typename T>
|
||||
constexpr bool is_identity_fop_v = is_identity_fop<T>::value;
|
||||
|
||||
template <int FIELD_ID>
|
||||
inline std::ostream& operator<<(std::ostream& out, Identity<FIELD_ID>)
|
||||
{
|
||||
return out << "Identity<" << FIELD_ID << ">";
|
||||
}
|
||||
|
||||
/// @brief Weight FieldOperator.
|
||||
///
|
||||
/// This FieldOperator is used to signal that this field contains the quadrature
|
||||
@@ -100,14 +90,6 @@ struct is_weight_fop : std::false_type {};
|
||||
template <>
|
||||
struct is_weight_fop<Weight> : std::true_type {};
|
||||
|
||||
template <typename T>
|
||||
constexpr bool is_weight_fop_v = is_weight_fop<T>::value;
|
||||
|
||||
inline std::ostream& operator<<(std::ostream& out, Weight)
|
||||
{
|
||||
return out << "Weight";
|
||||
}
|
||||
|
||||
/// @brief Value FieldOperator.
|
||||
///
|
||||
/// This FieldOperator is used to signal that the field contains the
|
||||
@@ -119,20 +101,11 @@ public:
|
||||
constexpr Value() : FieldOperator<FIELD_ID>() {};
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
template< typename T >
|
||||
struct is_value_fop : std::false_type {};
|
||||
|
||||
template <int T>
|
||||
struct is_value_fop<Value<T>> : std::true_type {};
|
||||
|
||||
template <typename T>
|
||||
constexpr bool is_value_fop_v = is_value_fop<T>::value;
|
||||
|
||||
template <int FIELD_ID>
|
||||
inline std::ostream& operator<<(std::ostream& out, Value<FIELD_ID>)
|
||||
{
|
||||
return out << "Value<" << FIELD_ID << ">";
|
||||
}
|
||||
struct is_value_fop<Value<FIELD_ID>> : std::true_type {};
|
||||
|
||||
/// @brief Gradient FieldOperator.
|
||||
///
|
||||
@@ -151,15 +124,6 @@ struct is_gradient_fop : std::false_type {};
|
||||
template <int FIELD_ID>
|
||||
struct is_gradient_fop<Gradient<FIELD_ID>> : std::true_type {};
|
||||
|
||||
template <typename T>
|
||||
constexpr bool is_gradient_fop_v = is_gradient_fop<T>::value;
|
||||
|
||||
template <int FIELD_ID>
|
||||
inline std::ostream& operator<<(std::ostream& out, Gradient<FIELD_ID>)
|
||||
{
|
||||
return out << "Gradient<" << FIELD_ID << ">";
|
||||
}
|
||||
|
||||
/// @brief Sum FieldOperator.
|
||||
///
|
||||
/// This FieldOperator is commonly used to signal that an output of a quadrature
|
||||
@@ -177,39 +141,4 @@ struct is_sum_fop : std::false_type {};
|
||||
template <int FIELD_ID>
|
||||
struct is_sum_fop<Sum<FIELD_ID>> : std::true_type {};
|
||||
|
||||
template <typename T>
|
||||
constexpr bool is_sum_fop_v = is_sum_fop<T>::value;
|
||||
|
||||
template <int FIELD_ID>
|
||||
inline std::ostream& operator<<(std::ostream& out, Sum<FIELD_ID>)
|
||||
{
|
||||
return out << "Sum<" << FIELD_ID << ">";
|
||||
}
|
||||
|
||||
/// @brief FunctionalValue FieldOperator.
|
||||
///
|
||||
/// This FieldOperator is commonly used to signal that an output of a quadrature
|
||||
/// function should be summed.
|
||||
template <int FIELD_ID = -1>
|
||||
class FunctionalValue : public FieldOperator<FIELD_ID>
|
||||
{
|
||||
public:
|
||||
constexpr FunctionalValue() : FieldOperator<FIELD_ID>() {};
|
||||
};
|
||||
|
||||
template< typename T >
|
||||
struct is_functionalvalue_fop : std::false_type {};
|
||||
|
||||
template <int FIELD_ID>
|
||||
struct is_functionalvalue_fop<FunctionalValue<FIELD_ID>> : std::true_type {};
|
||||
|
||||
template <typename T>
|
||||
constexpr bool is_functionalvalue_fop_v = is_functionalvalue_fop<T>::value;
|
||||
|
||||
template <int FIELD_ID>
|
||||
inline std::ostream& operator<<(std::ostream& out, FunctionalValue<FIELD_ID>)
|
||||
{
|
||||
return out << "FunctionalValue<" << FIELD_ID << ">";
|
||||
}
|
||||
|
||||
} // namespace mfem::future
|
||||
|
||||
@@ -0,0 +1,536 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "util.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
template <typename output_t>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_quadrature_data_to_fields_impl(
|
||||
DeviceTensor<2, real_t> &y,
|
||||
const DeviceTensor<3, real_t> &f,
|
||||
const output_t &output,
|
||||
const DofToQuadMap &dtq)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
|
||||
// assuming the quadrature point residual has to "play nice with
|
||||
// the test function"
|
||||
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [num_qp, cdim, num_dof] = B.GetShape();
|
||||
const int vdim = output.vdim > 0 ? output.vdim : cdim ;
|
||||
for (int dof = 0; dof < num_dof; dof++)
|
||||
{
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qp = 0; qp < num_qp; qp++)
|
||||
{
|
||||
acc += B(qp, 0, dof) * f(vd, 0, qp);
|
||||
}
|
||||
y(dof, vd) += acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (
|
||||
is_gradient_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [num_qp, dim, num_dof] = G.GetShape();
|
||||
const int vdim = output.vdim;
|
||||
for (int dof = 0; dof < num_dof; dof++)
|
||||
{
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int d = 0; d < dim; d++)
|
||||
{
|
||||
for (int qp = 0; qp < num_qp; qp++)
|
||||
{
|
||||
acc += G(qp, d, dof) * f(vd, d, qp);
|
||||
}
|
||||
}
|
||||
y(dof, vd) += acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (is_sum_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
// This is the "integral over all quadrature points type" applying
|
||||
// B = 1 s.t. B^T * C \in R^1.
|
||||
const auto [num_qp, unused, unused1] = B.GetShape();
|
||||
auto cc = Reshape(&f(0, 0, 0), num_qp);
|
||||
for (int i = 0; i < num_qp; i++)
|
||||
{
|
||||
y(0, 0) += cc(i);
|
||||
}
|
||||
}
|
||||
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [num_qp, unused, num_dof] = B.GetShape();
|
||||
const auto vdim = output.vdim;
|
||||
auto cc = Reshape(&f(0, 0, 0), num_qp * vdim);
|
||||
auto yy = Reshape(&y(0, 0), num_qp * vdim);
|
||||
for (int i = 0; i < num_qp * vdim; i++)
|
||||
{
|
||||
yy(i) = cc(i);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
|
||||
" for this field descriptor");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename output_t>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_quadrature_data_to_fields_tensor_impl_1d(
|
||||
DeviceTensor<2, real_t> &y,
|
||||
const DeviceTensor<3, real_t> &f,
|
||||
const output_t &output,
|
||||
const DofToQuadMap &dtq,
|
||||
std::array<DeviceTensor<1>, 6> &scratch_mem)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
|
||||
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = output.vdim;
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
|
||||
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, vdim);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qx = 0; qx < q1d; qx++)
|
||||
{
|
||||
acc += fqp(vd, 0, qx) * B(qx, 0, dx);
|
||||
}
|
||||
yd(dx, vd) = acc;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = G.GetShape();
|
||||
const int vdim = output.vdim;
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, vdim);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qx = 0; qx < q1d; qx++)
|
||||
{
|
||||
acc += fqp(vd, 0, qx) * G(qx, 0, dx);
|
||||
}
|
||||
yd(dx, vd) = acc;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
|
||||
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
|
||||
|
||||
for (int sq = 0; sq < output.size_on_qp; sq++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
yqp(sq, qx) = fqp(sq, qx);
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
|
||||
"for this field descriptor with sum factorization on"
|
||||
" tensor product elements");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename output_t>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_quadrature_data_to_fields_tensor_impl_2d(
|
||||
DeviceTensor<2, real_t> &y,
|
||||
const DeviceTensor<3, real_t> &f,
|
||||
const output_t &output,
|
||||
const DofToQuadMap &dtq,
|
||||
std::array<DeviceTensor<1>, 6> &scratch_mem)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
|
||||
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = output.vdim;
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
|
||||
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
|
||||
|
||||
auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qx = 0; qx < q1d; qx++)
|
||||
{
|
||||
acc += fqp(vd, 0, qx, qy) * B(qx, 0, dx);
|
||||
}
|
||||
s0(qy, dx) = acc;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qy = 0; qy < q1d; qy++)
|
||||
{
|
||||
acc += s0(qy, dx) * B(qy, 0, dy);
|
||||
}
|
||||
yd(dx, dy, vd) += acc;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = G.GetShape();
|
||||
const int vdim = output.vdim;
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
|
||||
|
||||
auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
|
||||
auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t uv[2] = {0.0, 0.0};
|
||||
for (int qx = 0; qx < q1d; qx++)
|
||||
{
|
||||
uv[0] += fqp(vd, 0, qx, qy) * G(qx, 0, dx);
|
||||
uv[1] += fqp(vd, 1, qx, qy) * B(qx, 0, dx);
|
||||
}
|
||||
s0(qy, dx) = uv[0];
|
||||
s1(qy, dx) = uv[1];
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t uv[2] = {0.0, 0.0};
|
||||
for (int qy = 0; qy < q1d; qy++)
|
||||
{
|
||||
uv[0] += s0(qy, dx) * B(qy, 0, dy);
|
||||
uv[1] += s1(qy, dx) * G(qy, 0, dy);
|
||||
}
|
||||
yd(dx, dy, vd) += uv[0] + uv[1];
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
|
||||
// // TODO: Check if this is the right fix for all cases
|
||||
// auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
|
||||
// auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
|
||||
// for (int sq = 0; sq < output.size_on_qp; sq++)
|
||||
// {
|
||||
// MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
// {
|
||||
// yqp(sq, qx) = fqp(sq, qx);
|
||||
// }
|
||||
// MFEM_SYNC_THREAD;
|
||||
// }
|
||||
|
||||
auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d);
|
||||
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
|
||||
|
||||
for (int sq = 0; sq < output.size_on_qp; sq++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
yqp(sq, qx, qy) = fqp(sq, qx, qy);
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
|
||||
" for this field descriptor with sum factorization on"
|
||||
" tensor product elements");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename output_t>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_quadrature_data_to_fields_tensor_impl_3d(
|
||||
DeviceTensor<2, real_t> &y,
|
||||
const DeviceTensor<3, real_t> &f,
|
||||
const output_t &output,
|
||||
const DofToQuadMap &dtq,
|
||||
std::array<DeviceTensor<1>, 6> &scratch_mem)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
|
||||
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = output.vdim;
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
|
||||
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
|
||||
|
||||
auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
|
||||
auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d, d1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qx = 0; qx < q1d; qx++)
|
||||
{
|
||||
acc += fqp(vd, 0, qx, qy, qz) * B(qx, 0, dx);
|
||||
}
|
||||
s0(qz, qy, dx) = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qy = 0; qy < q1d; qy++)
|
||||
{
|
||||
acc += s0(qz, qy, dx) * B(qy, 0, dy);
|
||||
}
|
||||
s1(qz, dy, dx) = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dz, z, d1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int qz = 0; qz < q1d; qz++)
|
||||
{
|
||||
acc += s1(qz, dy, dx) * B(qz, 0, dz);
|
||||
}
|
||||
yd(dx, dy, dz, vd) += acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = G.GetShape();
|
||||
const int vdim = output.vdim;
|
||||
const int test_dim = output.size_on_qp / vdim;
|
||||
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
|
||||
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
|
||||
|
||||
auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
|
||||
auto s1 = Reshape(&scratch_mem[1](0), q1d, q1d, d1d);
|
||||
auto s2 = Reshape(&scratch_mem[2](0), q1d, q1d, d1d);
|
||||
auto s3 = Reshape(&scratch_mem[3](0), q1d, d1d, d1d);
|
||||
auto s4 = Reshape(&scratch_mem[4](0), q1d, d1d, d1d);
|
||||
auto s5 = Reshape(&scratch_mem[5](0), q1d, d1d, d1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t uvw[3] = {0.0, 0.0, 0.0};
|
||||
for (int qx = 0; qx < q1d; qx++)
|
||||
{
|
||||
uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
|
||||
uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
|
||||
uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
|
||||
}
|
||||
s0(qz, qy, dx) = uvw[0];
|
||||
s1(qz, qy, dx) = uvw[1];
|
||||
s2(qz, qy, dx) = uvw[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t uvw[3] = {0.0, 0.0, 0.0};
|
||||
for (int qy = 0; qy < q1d; qy++)
|
||||
{
|
||||
uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
|
||||
uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
|
||||
uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
|
||||
}
|
||||
s3(qz, dy, dx) = uvw[0];
|
||||
s4(qz, dy, dx) = uvw[1];
|
||||
s5(qz, dy, dx) = uvw[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(dz, z, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dx, x, d1d)
|
||||
{
|
||||
real_t uvw[3] = {0.0, 0.0, 0.0};
|
||||
for (int qz = 0; qz < q1d; qz++)
|
||||
{
|
||||
uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
|
||||
uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
|
||||
uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
|
||||
}
|
||||
yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d, q1d);
|
||||
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
|
||||
|
||||
for (int sq = 0; sq < output.size_on_qp; sq++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
|
||||
" for this field descriptor with sum factorization on"
|
||||
" tensor product elements");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename output_t>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_quadrature_data_to_fields(
|
||||
DeviceTensor<2, real_t> &y,
|
||||
const DeviceTensor<3, real_t> &f,
|
||||
const output_t &output,
|
||||
const DofToQuadMap &dtq,
|
||||
std::array<DeviceTensor<1>, 6> &scratch_mem,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization)
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem);
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem);
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
map_quadrature_data_to_fields_tensor_impl_3d(y, f, output, dtq, scratch_mem);
|
||||
}
|
||||
else { MFEM_ABORT_KERNEL("dimension not supported"); }
|
||||
}
|
||||
else
|
||||
{
|
||||
map_quadrature_data_to_fields_impl(y, f, output, dtq);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mfem::future
|
||||
@@ -1,37 +0,0 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include <typeindex>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "../../general/array.hpp"
|
||||
#include "fielddescriptor.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
struct IntegratorContext
|
||||
{
|
||||
const ParMesh &mesh;
|
||||
const Array<int> *elem_attr;
|
||||
Array<int> attr;
|
||||
const int nentities;
|
||||
const std::vector<FieldDescriptor> &infds;
|
||||
const std::vector<FieldDescriptor> &outfds;
|
||||
const std::vector<FieldDescriptor> &unionfds;
|
||||
const IntegrationRule &ir;
|
||||
std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
|
||||
std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
|
||||
};
|
||||
|
||||
}
|
||||
@@ -0,0 +1,678 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "util.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
template <typename field_operator_t>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void map_field_to_quadrature_data_tensor_product_3d(
|
||||
DeviceTensor<2> &field_qp,
|
||||
const DofToQuadMap &dtq,
|
||||
const DeviceTensor<1> &field_e,
|
||||
const field_operator_t &input,
|
||||
const DeviceTensor<1, const real_t> &integration_weights,
|
||||
const std::array<DeviceTensor<1>, 6> &scratch_mem)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
|
||||
if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
|
||||
auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d, q1d);
|
||||
auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
|
||||
auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d, q1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dz, z, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dx = 0; dx < d1d; dx++)
|
||||
{
|
||||
acc += B(qx, 0, dx) * field(dx, dy, dz, vd);
|
||||
}
|
||||
s0(dz, dy, qx) = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(dz, z, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dy = 0; dy < d1d; dy++)
|
||||
{
|
||||
acc += s0(dz, dy, qx) * B(qy, 0, dy);
|
||||
}
|
||||
s1(dz, qy, qx) = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dz = 0; dz < d1d; dz++)
|
||||
{
|
||||
acc += s1(dz, qy, qx) * B(qz, 0, dz);
|
||||
}
|
||||
fqp(vd, qx, qy, qz) = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else if constexpr (
|
||||
is_gradient_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const int dim = input.dim;
|
||||
const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
|
||||
auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
|
||||
|
||||
auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
|
||||
auto s1 = Reshape(&scratch_mem[1](0), d1d, d1d, q1d);
|
||||
auto s2 = Reshape(&scratch_mem[2](0), d1d, q1d, q1d);
|
||||
auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
|
||||
auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dz, z, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t uv[2] = {0.0, 0.0};
|
||||
for (int dx = 0; dx < d1d; dx++)
|
||||
{
|
||||
const real_t f = field(dx, dy, dz, vd);
|
||||
uv[0] += f * B(qx, 0, dx);
|
||||
uv[1] += f * G(qx, 0, dx);
|
||||
}
|
||||
s0(dz, dy, qx) = uv[0];
|
||||
s1(dz, dy, qx) = uv[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(dz, z, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t uvw[3] = {0.0, 0.0, 0.0};
|
||||
for (int dy = 0; dy < d1d; dy++)
|
||||
{
|
||||
const real_t s0i = s0(dz, dy, qx);
|
||||
uvw[0] += s1(dz, dy, qx) * B(qy, 0, dy);
|
||||
uvw[1] += s0i * G(qy, 0, dy);
|
||||
uvw[2] += s0i * B(qy, 0, dy);
|
||||
}
|
||||
s2(dz, qy, qx) = uvw[0];
|
||||
s3(dz, qy, qx) = uvw[1];
|
||||
s4(dz, qy, qx) = uvw[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t uvw[3] = {0.0, 0.0, 0.0};
|
||||
for (int dz = 0; dz < d1d; dz++)
|
||||
{
|
||||
uvw[0] += s2(dz, qy, qx) * B(qz, 0, dz);
|
||||
uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
|
||||
uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
|
||||
}
|
||||
fqp(vd, 0, qx, qy, qz) = uvw[0];
|
||||
fqp(vd, 1, qx, qy, qz) = uvw[1];
|
||||
fqp(vd, 2, qx, qy, qz) = uvw[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
// TODO: Create separate function for clarity
|
||||
else if constexpr (
|
||||
std::is_same_v<std::decay_t<field_operator_t>, Weight>)
|
||||
{
|
||||
const int num_qp = integration_weights.GetShape()[0];
|
||||
// TODO: eeek
|
||||
const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
|
||||
auto w = Reshape(&integration_weights[0], q1d, q1d, q1d);
|
||||
auto f = Reshape(&field_qp[0], q1d, q1d, q1d);
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qz, z, q1d)
|
||||
{
|
||||
f(qx, qy, qz) = w(qx, qy, qz);
|
||||
}
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
const int q1d = B.GetShape()[0];
|
||||
auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d * q1d);
|
||||
field_qp = field;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
|
||||
"can't map field to quadrature data");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename field_operator_t>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void map_field_to_quadrature_data_tensor_product_2d(
|
||||
DeviceTensor<2> &field_qp,
|
||||
const DofToQuadMap &dtq,
|
||||
const DeviceTensor<1> &field_e,
|
||||
const field_operator_t &input,
|
||||
const DeviceTensor<1, const real_t> &integration_weights,
|
||||
const std::array<DeviceTensor<1>, 6> &scratch_mem)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
|
||||
if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
|
||||
auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d);
|
||||
auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dx = 0; dx < d1d; dx++)
|
||||
{
|
||||
acc += B(qx, 0, dx) * field(dx, dy, vd);
|
||||
}
|
||||
s0(dy, qx) = acc;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dy = 0; dy < d1d; dy++)
|
||||
{
|
||||
acc += s0(dy, qx) * B(qy, 0, dy);
|
||||
}
|
||||
fqp(vd, qx, qy) = acc;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
else if constexpr (
|
||||
is_gradient_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const int dim = input.dim;
|
||||
const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
|
||||
auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d);
|
||||
|
||||
auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
|
||||
auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(dy, y, d1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t uv[2] = {0.0, 0.0};
|
||||
for (int dx = 0; dx < d1d; dx++)
|
||||
{
|
||||
const real_t f = field(dx, dy, vd);
|
||||
uv[0] += f * B(qx, 0, dx);
|
||||
uv[1] += f * G(qx, 0, dx);
|
||||
}
|
||||
s0(dy, qx) = uv[0];
|
||||
s1(dy, qx) = uv[1];
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t uv[2] = {0.0, 0.0};
|
||||
for (int dy = 0; dy < d1d; dy++)
|
||||
{
|
||||
const real_t s0i = s0(dy, qx);
|
||||
uv[0] += s1(dy, qx) * B(qy, 0, dy);
|
||||
uv[1] += s0i * G(qy, 0, dy);
|
||||
}
|
||||
fqp(vd, 0, qx, qy) = uv[0];
|
||||
fqp(vd, 1, qx, qy) = uv[1];
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
// TODO: Create separate function for clarity
|
||||
else if constexpr (
|
||||
std::is_same_v<std::decay_t<field_operator_t>, Weight>)
|
||||
{
|
||||
const int num_qp = integration_weights.GetShape()[0];
|
||||
// TODO: eeek
|
||||
const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
|
||||
auto w = Reshape(&integration_weights[0], q1d, q1d);
|
||||
auto f = Reshape(&field_qp[0], q1d, q1d);
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qy, y, q1d)
|
||||
{
|
||||
f(qx, qy) = w(qx, qy);
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
const int q1d = B.GetShape()[0];
|
||||
auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d);
|
||||
field_qp = field;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
|
||||
"can't map field to quadrature data");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template <typename field_operator_t>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void map_field_to_quadrature_data_tensor_product_1d(
|
||||
DeviceTensor<2> &field_qp,
|
||||
const DofToQuadMap &dtq,
|
||||
const DeviceTensor<1> &field_e,
|
||||
const field_operator_t &input,
|
||||
const DeviceTensor<1, const real_t> &integration_weights,
|
||||
const std::array<DeviceTensor<1>, 6> &scratch_mem)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
|
||||
if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const auto field = Reshape(&field_e[0], d1d, vdim);
|
||||
auto fqp = Reshape(&field_qp[0], vdim, q1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dx = 0; dx < d1d; dx++)
|
||||
{
|
||||
acc += B(qx, 0, dx) * field(dx, vd);
|
||||
}
|
||||
fqp(vd, qx) = acc;
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
else if constexpr (
|
||||
is_gradient_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
const auto [q1d, unused, d1d] = B.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const int dim = input.dim;
|
||||
const auto field = Reshape(&field_e[0], d1d, vdim);
|
||||
auto fqp = Reshape(&field_qp[0], vdim, dim, q1d);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dx = 0; dx < d1d; dx++)
|
||||
{
|
||||
acc += G(qx, 0, dx) * field(dx, vd);
|
||||
}
|
||||
fqp(vd, 0, qx) = acc;
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
}
|
||||
// TODO: Create separate function for clarity
|
||||
else if constexpr (
|
||||
std::is_same_v<std::decay_t<field_operator_t>, Weight>)
|
||||
{
|
||||
const int num_qp = integration_weights.GetShape()[0];
|
||||
// TODO: eeek
|
||||
const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
|
||||
auto w = Reshape(&integration_weights[0], q1d);
|
||||
auto f = Reshape(&field_qp[0], q1d);
|
||||
MFEM_FOREACH_THREAD(qx, x, q1d)
|
||||
{
|
||||
f(qx) = w(qx);
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
|
||||
{
|
||||
const int q1d = B.GetShape()[0];
|
||||
auto field = Reshape(&field_e[0], input.size_on_qp, q1d);
|
||||
field_qp = field;
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
|
||||
"can't map field to quadrature data");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename field_operator_t>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_field_to_quadrature_data(
|
||||
DeviceTensor<2> field_qp,
|
||||
const DofToQuadMap &dtq,
|
||||
const DeviceTensor<1> &field_e,
|
||||
const field_operator_t &input,
|
||||
const DeviceTensor<1, const real_t> &integration_weights)
|
||||
{
|
||||
[[maybe_unused]] auto B = dtq.B;
|
||||
[[maybe_unused]] auto G = dtq.G;
|
||||
if constexpr (is_value_fop<field_operator_t>::value)
|
||||
{
|
||||
auto [num_qp, dim, num_dof] = B.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const auto field = Reshape(&field_e(0), num_dof, vdim);
|
||||
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
for (int qp = 0; qp < num_qp; qp++)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dof = 0; dof < num_dof; dof++)
|
||||
{
|
||||
acc += B(qp, 0, dof) * field(dof, vd);
|
||||
}
|
||||
field_qp(vd, qp) = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (is_gradient_fop<field_operator_t>::value)
|
||||
{
|
||||
const auto [num_qp, dim, num_dof] = G.GetShape();
|
||||
const int vdim = input.vdim;
|
||||
const auto field = Reshape(&field_e(0), num_dof, vdim);
|
||||
|
||||
auto f = Reshape(&field_qp[0], vdim, dim, num_qp);
|
||||
for (int vd = 0; vd < vdim; vd++)
|
||||
{
|
||||
for (int qp = 0; qp < num_qp; qp++)
|
||||
{
|
||||
for (int d = 0; d < dim; d++)
|
||||
{
|
||||
real_t acc = 0.0;
|
||||
for (int dof = 0; dof < num_dof; dof++)
|
||||
{
|
||||
acc += G(qp, d, dof) * field(dof, vd);
|
||||
}
|
||||
f(vd, d, qp) = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if constexpr (std::is_same_v<field_operator_t, Weight>)
|
||||
{
|
||||
const int num_qp = integration_weights.GetShape()[0];
|
||||
auto f = Reshape(&field_qp[0], num_qp);
|
||||
for (int qp = 0; qp < num_qp; qp++)
|
||||
{
|
||||
f(qp) = integration_weights(qp);
|
||||
}
|
||||
}
|
||||
else if constexpr (is_identity_fop<field_operator_t>::value)
|
||||
{
|
||||
auto [num_qp, unused, num_dof] = B.GetShape();
|
||||
const int size_on_qp = input.size_on_qp;
|
||||
const auto field = Reshape(&field_e[0], size_on_qp * num_qp);
|
||||
auto f = Reshape(&field_qp[0], size_on_qp * num_qp);
|
||||
for (int i = 0; i < size_on_qp * num_qp; i++)
|
||||
{
|
||||
f(i) = field(i);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(dfem::always_false<field_operator_t>,
|
||||
"can't map field to quadrature data");
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
template <typename field_operator_ts, size_t num_inputs, size_t num_fields>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void map_fields_to_quadrature_data(
|
||||
std::array<DeviceTensor<2>, num_inputs> &fields_qp,
|
||||
const std::array<DeviceTensor<1>, num_fields> &fields_e,
|
||||
const std::array<DofToQuadMap, num_inputs> &dtqmaps,
|
||||
const std::array<size_t, num_inputs> &input_to_field,
|
||||
const field_operator_ts &fops,
|
||||
const DeviceTensor<1, const real_t> &integration_weights,
|
||||
const std::array<DeviceTensor<1>, 6> &scratch_mem,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization = false)
|
||||
{
|
||||
// When the input_to_field map returns -1, this means the requested input
|
||||
// is the integration weight. Weights don't have a user defined field
|
||||
// attached to them and we create a dummy field which is not accessed
|
||||
// inside the functions it is passed to.
|
||||
const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
|
||||
for_constexpr<num_inputs>([&](auto i)
|
||||
{
|
||||
const DeviceTensor<1> &field_e =
|
||||
(input_to_field[i] == SIZE_MAX) ? dummy_field_weight :
|
||||
fields_e[input_to_field[i]];
|
||||
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_1d(
|
||||
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
|
||||
integration_weights, scratch_mem);
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_2d(
|
||||
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
|
||||
integration_weights, scratch_mem);
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_3d(
|
||||
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
|
||||
integration_weights, scratch_mem);
|
||||
}
|
||||
else
|
||||
{
|
||||
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
|
||||
MFEM_ABORT("unsupported dimension");
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
map_field_to_quadrature_data(
|
||||
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
|
||||
integration_weights);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
template <typename field_operator_t>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_field_to_quadrature_data_conditional(
|
||||
DeviceTensor<2> &field_qp,
|
||||
const DeviceTensor<1> &field_e,
|
||||
const DofToQuadMap &dtqmap,
|
||||
field_operator_t &fop,
|
||||
const DeviceTensor<1, const real_t> &integration_weights,
|
||||
const std::array<DeviceTensor<1>, 6> &scratch_mem,
|
||||
const bool &condition,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization = false)
|
||||
{
|
||||
if (condition)
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_1d(
|
||||
field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
|
||||
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_2d(
|
||||
field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_3d(
|
||||
field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
map_field_to_quadrature_data(
|
||||
field_qp, dtqmap, field_e, fop, integration_weights);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <size_t num_fields, size_t num_inputs, typename field_operator_ts>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_fields_to_quadrature_data_conditional(
|
||||
std::array<DeviceTensor<2>, num_inputs> &fields_qp,
|
||||
const std::array<DeviceTensor<1, const real_t>, num_fields> &fields_e,
|
||||
const std::array<DofToQuadMap, num_inputs> &dtqmaps,
|
||||
field_operator_ts fops,
|
||||
const DeviceTensor<1, const real_t> &integration_weights,
|
||||
const std::array<DeviceTensor<1>, 6> &scratch_mem,
|
||||
const std::array<bool, num_inputs> &conditions,
|
||||
const bool &use_sum_factorization = false)
|
||||
{
|
||||
for_constexpr<num_inputs>([&](auto i)
|
||||
{
|
||||
map_field_to_quadrature_data_conditional(
|
||||
fields_qp[i], fields_e[i], dtqmaps[i], get<i>(fops), integration_weights,
|
||||
scratch_mem, conditions[i], use_sum_factorization);
|
||||
});
|
||||
}
|
||||
|
||||
template <size_t num_inputs, typename field_operator_ts>
|
||||
MFEM_HOST_DEVICE
|
||||
void map_direction_to_quadrature_data_conditional(
|
||||
std::array<DeviceTensor<2>, num_inputs> &directions_qp,
|
||||
const DeviceTensor<1> &direction_e,
|
||||
const std::array<DofToQuadMap, num_inputs> &dtqmaps,
|
||||
field_operator_ts fops,
|
||||
const DeviceTensor<1, const real_t> &integration_weights,
|
||||
const std::array<DeviceTensor<1>, 6> &scratch_mem,
|
||||
const std::array<bool, num_inputs> &conditions,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization)
|
||||
{
|
||||
for_constexpr<num_inputs>([&](auto i)
|
||||
{
|
||||
if (conditions[i])
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_1d(
|
||||
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
|
||||
integration_weights, scratch_mem);
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_2d(
|
||||
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
|
||||
integration_weights, scratch_mem);
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
map_field_to_quadrature_data_tensor_product_3d(
|
||||
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
|
||||
integration_weights, scratch_mem);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
map_field_to_quadrature_data(
|
||||
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
|
||||
integration_weights);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -20,7 +20,7 @@ namespace mfem::future
|
||||
class ParameterSpace
|
||||
{
|
||||
public:
|
||||
ParameterSpace(int vdim = 1) : vdim(vdim) { dtq.FE = nullptr; dtq.IntRule = nullptr; }
|
||||
ParameterSpace(int vdim = 1) : vdim(vdim) {}
|
||||
|
||||
/// @brief Get vector dimension at each point
|
||||
///
|
||||
@@ -43,7 +43,7 @@ public:
|
||||
/// Get spatial dimension
|
||||
///
|
||||
/// returns always 1.
|
||||
constexpr int Dimension() const
|
||||
int Dimension() const
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
@@ -65,7 +65,7 @@ public:
|
||||
/// It should not be used by a user.
|
||||
///
|
||||
/// returns identity by default that is lazy evaluated.
|
||||
virtual const Operator* GetElementRestriction(ElementDofOrdering) const
|
||||
virtual const Operator* GetElementRestriction(ElementDofOrdering o) const
|
||||
{
|
||||
if (!elem_restr)
|
||||
{
|
||||
@@ -74,14 +74,11 @@ public:
|
||||
return elem_restr.get();
|
||||
}
|
||||
|
||||
virtual const Operator* GetB() const = 0;
|
||||
|
||||
virtual const Operator* GetBt() const = 0;
|
||||
|
||||
protected:
|
||||
int vdim;
|
||||
DofToQuad dtq;
|
||||
mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
|
||||
mutable std::unique_ptr<Operator> prolongation;
|
||||
mutable std::unique_ptr<Operator> elem_restr;
|
||||
};
|
||||
|
||||
/// @brief Uniform parameter space
|
||||
|
||||
@@ -0,0 +1,619 @@
|
||||
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
|
||||
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
|
||||
// LICENSE and NOTICE for details. LLNL-CODE-806117.
|
||||
//
|
||||
// This file is part of the MFEM library. For more information and source code
|
||||
// availability visit https://mfem.org.
|
||||
//
|
||||
// MFEM is free software; you can redistribute it and/or modify it under the
|
||||
// terms of the BSD-3 license. We welcome feedback and contributions, see file
|
||||
// CONTRIBUTING.md for details.
|
||||
#pragma once
|
||||
|
||||
#include "util.hpp"
|
||||
#include "qfunction_transform.hpp"
|
||||
|
||||
namespace mfem::future
|
||||
{
|
||||
|
||||
/// @brief Call a qfunction with the given parameters.
|
||||
///
|
||||
/// @param qfunc the qfunction to call.
|
||||
/// @param input_shmem the input shared memory.
|
||||
/// @param residual_shmem the residual shared memory.
|
||||
/// @param rs_qp the size of the residual.
|
||||
/// @param num_qp the number of quadrature points.
|
||||
/// @param q1d the number of quadrature points in 1D.
|
||||
/// @param dimension the spatial dimension.
|
||||
/// @param use_sum_factorization whether to use sum factorization.
|
||||
/// @tparam qf_param_ts the tuple type of the qfunction parameters.
|
||||
template <
|
||||
typename qf_param_ts,
|
||||
typename qfunc_t,
|
||||
std::size_t num_fields>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void call_qfunction(
|
||||
qfunc_t &qfunc,
|
||||
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
|
||||
DeviceTensor<2> &residual_shmem,
|
||||
const int &rs_qp,
|
||||
const int &num_qp,
|
||||
const int &q1d,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization)
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
|
||||
{
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
auto r = Reshape(&residual_shmem(0, q), rs_qp);
|
||||
apply_kernel(r, qfunc, qf_args, input_shmem, q);
|
||||
}
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
const int q = qx + q1d * qy;
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
auto r = Reshape(&residual_shmem(0, q), rs_qp);
|
||||
apply_kernel(r, qfunc, qf_args, input_shmem, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
auto r = Reshape(&residual_shmem(0, q), rs_qp);
|
||||
apply_kernel(r, qfunc, qf_args, input_shmem, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
|
||||
MFEM_ABORT("unsupported dimension for sum factorization");
|
||||
#endif
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
|
||||
{
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
auto r = Reshape(&residual_shmem(0, q), rs_qp);
|
||||
apply_kernel(r, qfunc, qf_args, input_shmem, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief Call a qfunction with the given parameters and
|
||||
/// compute it's derivative action.
|
||||
///
|
||||
/// @param qfunc the qfunction to call.
|
||||
/// @param input_shmem the input shared memory.
|
||||
/// @param shadow_shmem the shadow shared memory.
|
||||
/// @param residual_shmem the residual shared memory.
|
||||
/// @param das_qp the size of the derivative action.
|
||||
/// @param num_qp the number of quadrature points.
|
||||
/// @param q1d the number of quadrature points in 1D.
|
||||
/// @param dimension the spatial dimension.
|
||||
/// @param use_sum_factorization whether to use sum factorization.
|
||||
/// @tparam qf_param_ts the tuple type of the qfunction parameters.
|
||||
template <
|
||||
typename qf_param_ts,
|
||||
typename qfunc_t,
|
||||
std::size_t num_fields>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void call_qfunction_derivative_action(
|
||||
qfunc_t &qfunc,
|
||||
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
|
||||
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
|
||||
DeviceTensor<2> &residual_shmem,
|
||||
const int &das_qp,
|
||||
const int &num_qp,
|
||||
const int &q1d,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization)
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
|
||||
{
|
||||
auto r = Reshape(&residual_shmem(0, q), das_qp);
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
|
||||
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
|
||||
shadow_shmem, q);
|
||||
#else
|
||||
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
const int q = qx + q1d * qy;
|
||||
auto r = Reshape(&residual_shmem(0, q), das_qp);
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
|
||||
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
|
||||
shadow_shmem, q);
|
||||
#else
|
||||
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
auto r = Reshape(&residual_shmem(0, q), das_qp);
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
|
||||
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
|
||||
shadow_shmem, q);
|
||||
#else
|
||||
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("unsupported dimension");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
|
||||
{
|
||||
auto r = Reshape(&residual_shmem(0, q), das_qp);
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
|
||||
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
|
||||
shadow_shmem, q);
|
||||
#else
|
||||
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
|
||||
namespace detail
|
||||
{
|
||||
template <
|
||||
typename qf_param_ts,
|
||||
typename qfunc_t,
|
||||
std::size_t num_fields>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void call_qfunction_derivative(
|
||||
qfunc_t &qfunc,
|
||||
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
|
||||
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
|
||||
DeviceTensor<2> &residual_shmem,
|
||||
DeviceTensor<5> &qpdc,
|
||||
const DeviceTensor<1, const real_t> &itod,
|
||||
const int &das_qp,
|
||||
const int &q)
|
||||
{
|
||||
const int test_vdim = qpdc.GetShape()[0];
|
||||
const int test_op_dim = qpdc.GetShape()[1];
|
||||
const int trial_vdim = qpdc.GetShape()[2];
|
||||
const int num_qp = qpdc.GetShape()[4];
|
||||
const size_t num_inputs = itod.GetShape()[0];
|
||||
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
int m_offset = 0;
|
||||
for (size_t s = 0; s < num_inputs; s++)
|
||||
{
|
||||
const int trial_op_dim = static_cast<int>(itod(s));
|
||||
if (trial_op_dim == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
auto d_qp = Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
d_qp(j, m, q) = 1.0;
|
||||
|
||||
auto r = Reshape(&residual_shmem(0, q), das_qp);
|
||||
auto qf_args = decay_tuple<qf_param_ts> {};
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
|
||||
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
|
||||
shadow_shmem, q);
|
||||
#else
|
||||
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
|
||||
#endif
|
||||
d_qp(j, m, q) = 0.0;
|
||||
|
||||
auto f = Reshape(&r(0), test_vdim, test_op_dim);
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
qpdc(i, k, j, m + m_offset, q) = f(i, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
m_offset += trial_op_dim;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief Call a qfunction with the given parameters and
|
||||
/// compute it's derivative represented by the Jacobian on
|
||||
/// each quadrature point.
|
||||
///
|
||||
/// @param qfunc the qfunction to call.
|
||||
/// @param input_shmem the input shared memory.
|
||||
/// @param shadow_shmem the shadow shared memory.
|
||||
/// @param residual_shmem the residual shared memory.
|
||||
/// @param qpdc the quadrature point data cache holding the resulting
|
||||
/// Jacobians on each quadrature point.
|
||||
/// @param itod inputs trial operator dimension.
|
||||
/// If input is dependent the value corresponds to the spatial dimension, otherwise
|
||||
/// a zero indicates non-dependence on the variable.
|
||||
/// @param das_qp the size of the derivative action.
|
||||
/// @param q1d the number of quadrature points in 1D.
|
||||
/// @param dimension the spatial dimension.
|
||||
/// @param use_sum_factorization whether to use sum factorization.
|
||||
/// @tparam qf_param_ts the tuple type of the qfunction parameters.
|
||||
template <
|
||||
typename qf_param_ts,
|
||||
typename qfunc_t,
|
||||
std::size_t num_fields>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void call_qfunction_derivative(
|
||||
qfunc_t &qfunc,
|
||||
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
|
||||
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
|
||||
DeviceTensor<2> &residual_shmem,
|
||||
DeviceTensor<5> &qpdc,
|
||||
const DeviceTensor<1, const real_t> &itod,
|
||||
const int &das_qp,
|
||||
const int &q1d,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization)
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
|
||||
{
|
||||
detail::call_qfunction_derivative<qf_param_ts>(
|
||||
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
|
||||
}
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
const int q = qx + q1d * qy;
|
||||
detail::call_qfunction_derivative<qf_param_ts>(
|
||||
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
detail::call_qfunction_derivative<qf_param_ts>(
|
||||
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("unsupported dimension");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const int num_qp = qpdc.GetShape()[4];
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
|
||||
{
|
||||
detail::call_qfunction_derivative<qf_param_ts>(
|
||||
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
|
||||
}
|
||||
}
|
||||
MFEM_SYNC_THREAD;
|
||||
}
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
/// @brief Apply the quadrature point data cache (qpdc) to a vector
|
||||
/// (usually a direction) on quadrature point q.
|
||||
///
|
||||
/// The qpdc consists of compatible data to be used for integration with a test
|
||||
/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
|
||||
/// function including integration weights and necessesary transformations.
|
||||
///
|
||||
/// @param fhat the qpdc applied to a vector in shadow_memory.
|
||||
/// @param shadow_shmem the shadow shared memory.
|
||||
/// @param qpdc the quadrature point data cache holding the resulting
|
||||
/// Jacobians on each quadrature point.
|
||||
/// @param itod inputs trial operator dimension.
|
||||
/// If input is dependent the value corresponds to the spatial dimension, otherwise
|
||||
/// a zero indicates non-dependence on the variable.
|
||||
/// @param q the current quadrature point index.
|
||||
template <size_t num_fields>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void apply_qpdc(
|
||||
DeviceTensor<3> &fhat,
|
||||
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
|
||||
const DeviceTensor<5, const real_t> &qpdc,
|
||||
const DeviceTensor<1, const real_t> &itod,
|
||||
const int &q)
|
||||
{
|
||||
const int test_vdim = qpdc.GetShape()[0];
|
||||
const int test_op_dim = qpdc.GetShape()[1];
|
||||
const int trial_vdim = qpdc.GetShape()[2];
|
||||
const int num_qp = qpdc.GetShape()[4];
|
||||
const size_t num_inputs = itod.GetShape()[0];
|
||||
|
||||
for (int i = 0; i < test_vdim; i++)
|
||||
{
|
||||
for (int k = 0; k < test_op_dim; k++)
|
||||
{
|
||||
real_t sum = 0.0;
|
||||
int m_offset = 0;
|
||||
for (size_t s = 0; s < num_inputs; s++)
|
||||
{
|
||||
const int trial_op_dim = static_cast<int>(itod(s));
|
||||
if (trial_op_dim == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
const auto d_qp =
|
||||
Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
|
||||
for (int j = 0; j < trial_vdim; j++)
|
||||
{
|
||||
for (int m = 0; m < trial_op_dim; m++)
|
||||
{
|
||||
sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
|
||||
}
|
||||
}
|
||||
m_offset += trial_op_dim;
|
||||
}
|
||||
fhat(i, k, q) = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief Apply the quadrature point data cache (qpdc) to a vector
|
||||
/// (usually a direction).
|
||||
///
|
||||
/// The qpdc consists of compatible data to be used for integration with a test
|
||||
/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
|
||||
/// function including integration weights and necessesary transformations.
|
||||
///
|
||||
/// @param fhat the qpdc applied to a vector in shadow_memory.
|
||||
/// @param shadow_shmem the shadow shared memory.
|
||||
/// @param qpdc the quadrature point data cache holding the resulting
|
||||
/// Jacobians on each quadrature point.
|
||||
/// @param itod inputs trial operator dimension.
|
||||
/// If input is dependent the value corresponds to the spatial dimension, otherwise
|
||||
/// a zero indicates non-dependence on the variable.
|
||||
/// @param q1d number of quadrature points in 1D.
|
||||
/// @param dimension spatial dimension.
|
||||
/// @param use_sum_factorization whether to use sum factorization.
|
||||
template <size_t num_fields>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void apply_qpdc(
|
||||
DeviceTensor<3> &fhat,
|
||||
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
|
||||
const DeviceTensor<5, const real_t> &qpdc,
|
||||
const DeviceTensor<1, const real_t> &itod,
|
||||
const int &q1d,
|
||||
const int &dimension,
|
||||
const bool &use_sum_factorization)
|
||||
{
|
||||
if (use_sum_factorization)
|
||||
{
|
||||
if (dimension == 1)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
|
||||
{
|
||||
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
|
||||
}
|
||||
}
|
||||
else if (dimension == 2)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
const int q = qx + q1d * qy;
|
||||
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (dimension == 3)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
|
||||
{
|
||||
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
|
||||
{
|
||||
const int q = qx + q1d * (qy + q1d * qz);
|
||||
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
MFEM_ABORT_KERNEL("unsupported dimension");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const int num_qp = qpdc.GetShape()[4];
|
||||
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
|
||||
{
|
||||
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename qfunc_t, typename args_ts, size_t num_args>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void apply_kernel(
|
||||
DeviceTensor<1, real_t> &f_qp,
|
||||
const qfunc_t &qfunc,
|
||||
args_ts &args,
|
||||
const std::array<DeviceTensor<2>, num_args> &u,
|
||||
int qp)
|
||||
{
|
||||
process_qf_args(u, args, qp);
|
||||
process_qf_result(f_qp, get<0>(apply(qfunc, args)));
|
||||
}
|
||||
|
||||
template <typename qfunc_t, typename arg_ts, size_t num_args>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void apply_kernel_native_dual(
|
||||
DeviceTensor<1, real_t> &f_qp,
|
||||
const qfunc_t &qfunc,
|
||||
arg_ts &args,
|
||||
const std::array<DeviceTensor<2>, num_args> &u,
|
||||
const std::array<DeviceTensor<2>, num_args> &v,
|
||||
const int &qp_idx)
|
||||
{
|
||||
process_qf_args(u, v, args, qp_idx);
|
||||
auto r = get<0>(apply(qfunc, args));
|
||||
process_derivative_from_native_dual(f_qp, r);
|
||||
}
|
||||
|
||||
#ifdef MFEM_USE_ENZYME
|
||||
|
||||
template <typename func_t, typename... arg_ts>
|
||||
MFEM_HOST_DEVICE inline
|
||||
auto qfunction_wrapper(const func_t &f, arg_ts &&...args)
|
||||
{
|
||||
return f(args...);
|
||||
}
|
||||
|
||||
// Version for active function arguments only
|
||||
//
|
||||
// This is an Enzyme regression and can be removed in later versions.
|
||||
template <typename qfunc_t, typename arg_ts, std::size_t... Is,
|
||||
typename inactive_arg_ts>
|
||||
MFEM_HOST_DEVICE inline
|
||||
auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
|
||||
arg_ts &&shadow_args,
|
||||
std::index_sequence<Is...>,
|
||||
inactive_arg_ts &&inactive_args,
|
||||
std::index_sequence<>)
|
||||
{
|
||||
using qf_return_t = typename create_function_signature<
|
||||
decltype(&qfunc_t::operator())>::type::return_t;
|
||||
return __enzyme_fwddiff<qf_return_t>(
|
||||
qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...>, enzyme_const,
|
||||
(void *)&qfunc, enzyme_dup, &get<Is>(args)..., enzyme_interleave,
|
||||
&get<Is>(shadow_args)...);
|
||||
}
|
||||
|
||||
// Interleave function arguments for enzyme
|
||||
template <typename qfunc_t, typename arg_ts, std::size_t... Is,
|
||||
typename inactive_arg_ts, std::size_t... Js>
|
||||
MFEM_HOST_DEVICE inline
|
||||
auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
|
||||
arg_ts &&shadow_args,
|
||||
std::index_sequence<Is...>,
|
||||
inactive_arg_ts &&inactive_args,
|
||||
std::index_sequence<Js...>)
|
||||
{
|
||||
using qf_return_t = typename create_function_signature<
|
||||
decltype(&qfunc_t::operator())>::type::return_t;
|
||||
return __enzyme_fwddiff<qf_return_t>(
|
||||
qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...,
|
||||
decltype(get<Js>(inactive_args))...>,
|
||||
enzyme_const, (void *)&qfunc, enzyme_dup, &get<Is>(args)...,
|
||||
enzyme_const, &get<Js>(inactive_args)..., enzyme_interleave,
|
||||
&get<Is>(shadow_args)...);
|
||||
}
|
||||
|
||||
template <typename qfunc_t, typename arg_ts, typename inactive_arg_ts>
|
||||
MFEM_HOST_DEVICE inline
|
||||
auto fwddiff_apply_enzyme(qfunc_t &qfunc, arg_ts &&args,
|
||||
arg_ts &&shadow_args,
|
||||
inactive_arg_ts &&inactive_args)
|
||||
{
|
||||
auto arg_indices = std::make_index_sequence<
|
||||
tuple_size<std::remove_reference_t<arg_ts>>::value> {};
|
||||
|
||||
auto inactive_arg_indices = std::make_index_sequence<
|
||||
tuple_size<std::remove_reference_t<inactive_arg_ts>>::value> {};
|
||||
|
||||
return fwddiff_apply_enzyme_indexed(qfunc, args, shadow_args, arg_indices,
|
||||
inactive_args, inactive_arg_indices);
|
||||
}
|
||||
|
||||
template <typename qfunc_t, typename arg_ts, size_t num_args>
|
||||
MFEM_HOST_DEVICE inline
|
||||
void apply_kernel_fwddiff_enzyme(
|
||||
DeviceTensor<1, real_t> &f_qp,
|
||||
qfunc_t &qfunc,
|
||||
arg_ts &args,
|
||||
arg_ts &shadow_args,
|
||||
const std::array<DeviceTensor<2>, num_args> &u,
|
||||
const std::array<DeviceTensor<2>, num_args> &v,
|
||||
int qp_idx)
|
||||
{
|
||||
process_qf_args(u, args, qp_idx);
|
||||
process_qf_args(v, shadow_args, qp_idx);
|
||||
process_qf_result(f_qp,
|
||||
get<0>(fwddiff_apply_enzyme(qfunc, args, shadow_args, tuple<> {})));
|
||||
}
|
||||
#endif // MFEM_USE_ENZYME
|
||||
|
||||
} // namespace mfem::future
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user