diff --git a/.github/workflows/builds-and-tests.yml b/.github/workflows/builds-and-tests.yml index f33d6b55cb..231f768f3f 100644 --- a/.github/workflows/builds-and-tests.yml +++ b/.github/workflows/builds-and-tests.yml @@ -132,12 +132,14 @@ jobs: hypre-target: int32 precision: fp64 enzyme: true - config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme) + config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme) LDFLAGS=-L$LLVM_PREFIX/lib/c++ name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }} runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.enzyme && true || false }} + steps: # Fix 'No space left on device' errors for Ubuntu builds. - name: Run Actions Cleaner @@ -168,10 +170,13 @@ jobs: env shell: bash + # For info on Xcode see: + # - https://github.com/actions/runner-images/issues/12541 + # - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode - name: Xcode version setup (MacOS) if: matrix.os == 'macos-latest' run: | - XCODE_PATH="/Applications/Xcode_15.3.app" + XCODE_PATH="/Applications/Xcode_16.4.app" echo "> sudo xcode-select -s ${XCODE_PATH}" sudo xcode-select -s ${XCODE_PATH} echo "> g++ -v" @@ -289,10 +294,12 @@ jobs: run: | export HOMEBREW_NO_INSTALL_CLEANUP=1 brew update - brew install llvm@20 enzyme - echo "LLVM_PREFIX=$(brew --prefix llvm@20)" >> $GITHUB_ENV - echo "OMPI_CC=$(brew --prefix llvm@20)/bin/clang" >> $GITHUB_ENV - echo "OMPI_CXX=$(brew --prefix llvm@20)/bin/clang++" >> $GITHUB_ENV + brew install enzyme + ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p') + LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM) + echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV + echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV + echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV # MFEM build and test - name: build diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 7e5cd906fc..5dba26c75e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -29,3 +29,47 @@ jobs: operations-per-run: 500 exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next" exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next" + + # Stale action for PRs with "in-review" label. + stale-in-review-pr: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + actions: write + + steps: + - uses: actions/stale@v9 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 150 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.' + only-pr-labels: "in-review" + days-before-pr-stale: 150 + days-before-pr-close: 30 + days-before-issue-stale: -1 + days-before-issue-close: -1 + stale-pr-label: 'stale' + operations-per-run: 500 + + # Stale action for PRs with "WIP" label. + stale-wip-pr: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + actions: write + + steps: + - uses: actions/stale@v9 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 300 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.' + only-pr-labels: "WIP" + days-before-pr-stale: 300 + days-before-pr-close: 30 + days-before-issue-stale: -1 + days-before-issue-close: -1 + stale-pr-label: 'stale' + operations-per-run: 500 diff --git a/.gitignore b/.gitignore index cc6084382a..332f1bee9f 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ examples/sol_u.* examples/sol_p.* examples/sol_r.* examples/sol_i.* +examples/sol_z.* examples/ex6p-checkpoint.* examples/order.* examples/ex9.mesh @@ -208,10 +209,13 @@ miniapps/electromagnetics/volta miniapps/electromagnetics/tesla miniapps/electromagnetics/maxwell miniapps/electromagnetics/joule +miniapps/electromagnetics/lorentz miniapps/electromagnetics/Volta-AMR* miniapps/electromagnetics/Tesla-AMR* miniapps/electromagnetics/Maxwell-Parallel* miniapps/electromagnetics/Joule_[0-9]* +miniapps/electromagnetics/Lorentz_[0-9]* +miniapps/electromagnetics/Lorentz.dat miniapps/gslib/field-diff miniapps/gslib/field-interp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df1a278e78..26beecb342 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,91 +9,550 @@ # terms of the BSD-3 license. We welcome feedback and contributions, see file # CONTRIBUTING.md for details. +# DESCRIPTION: +############################################################################### # General GitLab pipelines configurations for supercomputers and Linux clusters -# at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is -# LLNL-specific! - -include: - - project: 'lc-templates/id_tokens' - file: 'id_tokens.yml' - -# The pipeline is divided into stages. Usually, jobs in a given stage wait for -# the preceding stages to complete before to start. However, we sometimes use -# the "needs" keyword and express the DAG of jobs for more efficiency. -# - We use setup and setup_baseline phases to download content outside of mfem -# directory. -# - Allocate/Release is where ruby resource are allocated/released once for all. -# - Build and Test is where we build and MFEM for multiple toolchains. -# - Baseline_checks gathers baseline-type test suites execution -# - Baseline_publish, only available on master, allows to update baseline -# results -stages: - - sub-pipelines +# at Lawrence Livermore National Laboratory (LLNL). +# This entire pipeline is LLNL-specific +# +# Important note: This file is a template provided by llnl/radiuss-shared-ci. +# Remains to set variable values, change the reference to the radiuss-shared-ci +# repo, opt-in and out optional features. The project can then extend it with +# additional stages. +# +# In addition, each project should copy over and complete: +# - .gitlab/custom-jobs-and-variables.yml +# - .gitlab/subscribed-pipelines.yml +# +# The jobs should be specified in a file local to the project, +# - .gitlab/jobs/${CI_MACHINE}.yml +# or generated (see LLNL/Umpire for an example). +############################################################################### +# MAP OF GITLAB CI +####################### +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# File dependencies: direct, through jobs, through variables +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# .gitlab-ci.yml +# ├── .build-and-test [job] +# │ ├── .gitlab/custom-jobs-and-variables.yml +# │ │ ├── .custom_job [job] +# │ │ ├── .reproducer_vars [job] +# │ │ ├── .report_job_success [job] +# │ │ │ └── .gitlab/scripts/report_build_and_test [script] +# │ │ │ ├── .gitlab/scripts/safe_create_rundir [script] +# │ │ │ └── .gitlab/scripts/git_try_to_push [script] +# │ │ ├── .report_job_failure [job] +# │ │ │ └── .gitlab/scripts/report_build_and_test [script] +# │ │ │ ├── .gitlab/scripts/safe_create_rundir [script] +# │ │ │ └── .gitlab/scripts/git_try_to_push [script] +# │ │ └── JOB_CMD [var] +# │ │ └── tests/gitlab/build_and_test [script] +# │ │ └── tests/gitlab/get_mfem_uberenv [script] +# │ ├── /pipelines/matrix.yml [conditional] +# │ │ ├── .on_matrix [job] +# │ │ ├── .matrix_reproducer_init [job] +# │ │ ├── .matrix_reproducer_vars [job] +# │ │ ├── .matrix_reproducer_job [job] +# │ │ ├── .matrix_job_command [job] +# │ │ └── .job_on_matrix [job] +# │ ├── /pipelines/dane.yml [conditional] +# │ │ ├── .on_dane [job] +# │ │ ├── .dane_reproducer_init [job] +# │ │ ├── .dane_reproducer_vars [job] +# │ │ ├── .dane_reproducer_job [job] +# │ │ ├── .dane_job_command [job] +# │ │ ├── .job_on_dane [job] +# │ │ ├── allocate_resources [job] +# │ │ └── release_resources [job] +# │ ├── /pipelines/tioga.yml [conditional] +# │ │ ├── .on_tioga [job] +# │ │ ├── .tioga_reproducer_init [job] +# │ │ ├── .tioga_reproducer_vars [job] +# │ │ ├── .tioga_reproducer_job [job] +# │ │ ├── .tioga_job_command [job] +# │ │ ├── .job_on_tioga [job] +# │ │ ├── allocate_resources [job] +# │ │ └── release_resources [job] +# │ ├── /matrix-jobs.yml [conditional, from 'generate-job-lists'] +# │ │ ├── .gitlab/jobs/matrix.yml +# │ │ │ ├── .matrix_reproducer_vars [job] +# │ │ │ ├── setup [job] +# │ │ │ │ └── ./tests/gitlab/build_and_test_setup [script] +# │ │ │ ├── opt_mpi_cuda_gcc [job] +# │ │ │ └── opt_mpi_cuda_hypre_cuda_gcc [job] +# │ │ └── .gitlab/jobs/matrix-reports.yml [used conditionally] +# │ │ ├── report_job_success +# │ │ └── report_job_failure +# │ ├── /dane-jobs.yml [conditional, from 'generate-job-lists'] +# │ │ ├── .gitlab/jobs/dane.yml +# │ │ │ ├── .dane_reproducer_vars [job] +# │ │ │ ├── setup [job] +# │ │ │ │ └── ./tests/gitlab/build_and_test_setup [script] +# │ │ │ ├── debug_ser_gcc_10 [job] +# │ │ │ ├── debug_par_gcc_10 [job] +# │ │ │ ├── opt_ser_gcc_10 [job] +# │ │ │ ├── opt_par_gcc_10 [job] +# │ │ │ ├── opt_par_gcc_10_sundials [job] +# │ │ │ ├── opt_par_gcc_10_petsc [job] +# │ │ │ └── opt_par_gcc_10_pumi [job] +# │ │ └── .gitlab/jobs/dane-reports.yml [used conditionally] +# │ │ ├── report_job_success +# │ │ └── report_job_failure +# │ └── /tioga-jobs.yml [conditional, from 'generate-job-lists'] +# │ ├── .gitlab/jobs/tioga.yml +# │ │ ├── .tioga_reproducer_vars [job] +# │ │ ├── setup [job] +# │ │ │ └── ./tests/gitlab/build_and_test_setup [script] +# │ │ └── cce_16_0_1 [job] +# │ └── .gitlab/jobs/tioga-reports.yml [used conditionally] +# │ ├── report_job_success +# │ └── report_job_failure +# └── .gitlab/subscribed-pipelines.yml +# ├── .machine-check [job] +# ├── generate-job-lists [job] +# ├── dane-up-check [job] +# ├── dane-build-and-test [job] +# ├── dane-baseline [job] +# │ └── .gitlab/dane-baseline.yml +# │ ├── .on_dane [job] +# │ ├── baselinecheck_mfem_intel_dane [job] +# │ │ └── .gitlab/scripts/baseline [script] +# │ ├── cleanup [job] +# │ ├── report_baseline [job] +# │ │ ├── .gitlab/scripts/safe_create_rundir [script] +# │ │ └── .gitlab/scripts/git_try_to_push [script] +# │ ├── baselinepublish_mfem_dane [job] +# │ │ └── .gitlab/scripts/rebaseline [script] +# │ ├── .gitlab/custom-jobs-and-variables.yml +# │ │ └── +# │ └── .gitlab/configs/setup-baseline.yml +# │ └── setup_baseline [job] +# ├── tioga-up-check [job] +# ├── tioga-build-and-test [job] +# ├── matrix-up-check [job] +# └── matrix-build-and-test [job] +# +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# File tree hierarchy with file contents highlights +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# In addition to the files in the MFEM repo, the Gitlab CI uses files from the +# radiuss/radiuss-shared-ci project, see below, after the tree. +# +# +# ├── .gitlab-ci.yml [this file] +# │ ├── +# │ │ └── .build-and-test +# │ ├── +# │ │ ├── .gitlab/subscribed-pipelines.yml +# │ │ ├── .gitlab/custom-jobs-and-variables.yml [by ".build-and-test"] +# │ │ ├── [by ".build-and-test"] +# │ │ │ ├── artifact: '${CI_MACHINE}-jobs.yml' +# │ │ │ └── job: 'generate-job-lists' +# │ │ └── [by ".build-and-test"] +# │ │ ├── project: 'radiuss/radiuss-shared-ci' +# │ │ ├── ref: 'v2025.09.1' +# │ │ └── file: 'pipelines/${CI_MACHINE}.yml' +# │ └── +# │ ├── CUSTOM_CI_BUILDS_DIR +# │ ├── USER_CI_TOP_DIR +# │ ├── SHARED_REPOS_DIR +# │ ├── AUTOTEST_ROOT +# │ ├── MFEM_DATA_DIR +# │ ├── AUTOTEST +# │ ├── AUTOTEST_COMMIT +# │ ├── REBASELINE +# │ ├── GITHUB_PROJECT_NAME +# │ └── GITHUB_PROJECT_ORG +# ├── .gitlab +# │ ├── configs +# │ │ └── setup-baseline.yml +# │ │ ├── +# │ │ │ └── setup_baseline +# │ │ └── +# │ │ ├── MACHINE_NAME +# │ │ ├── REBASELINE +# │ │ ├── AUTOTEST +# │ │ ├── AUTOTEST_COMMIT +# │ │ ├── BUILD_ROOT +# │ │ ├── TPLS_REPO +# │ │ ├── TESTS_REPO +# │ │ ├── AUTOTEST_ROOT +# │ │ └── AUTOTEST_REPO +# │ ├── jobs +# │ │ ├── matrix-reports.yml +# │ │ │ ├── +# │ │ │ │ ├── report_job_success +# │ │ │ │ └── report_job_failure +# │ │ │ └── +# │ │ │ ├── .on_matrix +# │ │ │ ├── .report_job_success +# │ │ │ └── .report_job_failure +# │ │ ├── matrix.yml +# │ │ │ ├── +# │ │ │ │ ├── .matrix_reproducer_vars +# │ │ │ │ ├── setup +# │ │ │ │ ├── opt_mpi_cuda_gcc +# │ │ │ │ └── opt_mpi_cuda_hypre_cuda_gcc +# │ │ │ ├── +# │ │ │ │ ├── .reproducer_vars +# │ │ │ │ ├── .on_matrix +# │ │ │ │ └── .job_on_matrix +# │ │ │ ├── +# │ │ │ │ └── tests/gitlab/build_and_test_setup [by "setup"] +# │ │ │ └── +# │ │ │ └── SPEC +# │ │ ├── dane-reports.yml +# │ │ │ ├── +# │ │ │ │ ├── report_job_success +# │ │ │ │ └── report_job_failure +# │ │ │ └── +# │ │ │ ├── .on_dane +# │ │ │ ├── .report_job_success +# │ │ │ └── .report_job_failure +# │ │ ├── dane.yml +# │ │ │ ├── +# │ │ │ │ ├── .dane_reproducer_vars +# │ │ │ │ ├── setup +# │ │ │ │ ├── debug_ser_gcc_10 +# │ │ │ │ ├── debug_par_gcc_10 +# │ │ │ │ ├── opt_ser_gcc_10 +# │ │ │ │ ├── opt_par_gcc_10 +# │ │ │ │ ├── opt_par_gcc_10_sundials +# │ │ │ │ ├── opt_par_gcc_10_petsc +# │ │ │ │ └── opt_par_gcc_10_pumi +# │ │ │ ├── +# │ │ │ │ ├── .reproducer_vars +# │ │ │ │ ├── .on_dane +# │ │ │ │ └── .job_on_dane +# │ │ │ ├── +# │ │ │ │ └── tests/gitlab/build_and_test_setup [by "setup"] +# │ │ │ └── +# │ │ │ ├── SPEC +# │ │ │ └── THREADS +# │ │ ├── tioga-reports.yml +# │ │ │ ├── +# │ │ │ │ ├── report_job_success +# │ │ │ │ └── report_job_failure +# │ │ │ └── +# │ │ │ ├── .on_tioga +# │ │ │ ├── .report_job_success +# │ │ │ └── .report_job_failure +# │ │ └── tioga.yml +# │ │ ├── +# │ │ │ ├── .tioga_reproducer_vars +# │ │ │ ├── setup +# │ │ │ └── opt_mpi_rocm_hypre_rocm +# │ │ ├── +# │ │ │ ├── .reproducer_vars +# │ │ │ ├── .on_tioga +# │ │ │ └── .job_on_tioga +# │ │ ├── +# │ │ │ └── tests/gitlab/build_and_test_setup [by "setup"] +# │ │ └── +# │ │ ├── SPEC +# │ │ └── THREADS +# │ ├── scripts +# │ │ ├── baseline +# │ │ │ └── +# │ │ │ ├── BASELINE_TEST +# │ │ │ ├── SYS_TYPE +# │ │ │ ├── MACHINE_NAME +# │ │ │ ├── CI_PROJECT_DIR +# │ │ │ ├── ARTIFACTS_DIR +# │ │ │ ├── BUILD_ROOT +# │ │ │ └── TPLS_DIR +# │ │ ├── git_try_to_push +# │ │ ├── rebaseline +# │ │ │ └── +# │ │ │ ├── CI_PROJECT_DIR +# │ │ │ ├── ARTIFACTS_DIR +# │ │ │ ├── SYS_TYPE +# │ │ │ ├── BUILD_ROOT +# │ │ │ ├── MACHINE_NAME +# │ │ │ └── CI_PIPELINE_ID +# │ │ ├── report_build_and_test +# │ │ │ ├── +# │ │ │ │ ├── .gitlab/scripts/safe_create_rundir +# │ │ │ │ └── .gitlab/scripts/git_try_to_push +# │ │ │ └── +# │ │ │ ├── AUTOTEST_ROOT +# │ │ │ ├── CI_COMMIT_REF_SLUG +# │ │ │ ├── CI_PROJECT_DIR +# │ │ │ ├── CI_PIPELINE_URL +# │ │ │ ├── AUTOTEST_COMMIT +# │ │ │ └── CI_MACHINE +# │ │ └── safe_create_rundir +# │ ├── custom-jobs-and-variables.yml +# │ │ ├── +# │ │ │ ├── .custom_job +# │ │ │ ├── .reproducer_vars +# │ │ │ ├── .report_job_success +# │ │ │ └── .report_job_failure +# │ │ ├── +# │ │ │ ├── tests/gitlab/build_and_test [in JOB_CMD] +# │ │ │ └── .gitlab/scripts/report_build_and_test [by .report_job_*] +# │ │ ├── +# │ │ │ ├── JOB_CMD +# │ │ │ ├── BUILD_ROOT +# │ │ │ ├── ALLOC_NAME +# │ │ │ ├── TPLS_REPO +# │ │ │ ├── TESTS_REPO +# │ │ │ ├── AUTOTEST_REPO +# │ │ │ ├── MFEM_DATA_REPO +# │ │ │ ├── ARTIFACTS_DIR: artifacts +# │ │ │ ├── SLURM_OVERLAP: 1 +# │ │ │ ├── DANE_SHARED_ALLOC +# │ │ │ ├── DANE_JOB_ALLOC +# │ │ │ ├── TIOGA_SHARED_ALLOC +# │ │ │ ├── TIOGA_JOB_ALLOC +# │ │ │ └── MATRIX_JOB_ALLOC +# │ │ └── +# │ │ ├── SPEC +# │ │ ├── BUILD_ROOT +# │ │ └── ... +# │ ├── dane-baseline.yml +# │ │ ├── +# │ │ │ ├── .on_dane +# │ │ │ ├── baselinecheck_mfem_intel_dane +# │ │ │ ├── cleanup +# │ │ │ ├── report_baseline +# │ │ │ └── baselinepublish_mfem_dane +# │ │ ├── +# │ │ │ ├── .gitlab/custom-jobs-and-variables.yml +# │ │ │ ├── .gitlab/configs/setup-baseline.yml +# │ │ │ ├── .gitlab/scripts/rebaseline +# │ │ │ ├── .gitlab/scripts/baseline +# │ │ │ └── .gitlab/scripts/git_try_to_push +# │ │ ├── +# │ │ │ ├── BASELINE_TEST: baseline +# │ │ │ ├── MACHINE_NAME: dane +# │ │ │ ├── TPLS_DIR +# │ │ │ └── export MFEM_TEST_NP +# │ │ └── +# │ │ ├── ON_DANE +# │ │ ├── AUTOTEST [defined by .gitlab-ci.yml] +# │ │ ├── BUILD_ROOT [defined by custom-jobs-and-variables.yml] +# │ │ ├── TPLS_DIR [defined by this file] +# │ │ ├── ARTIFACTS_DIR [defined by custom-jobs-and-variables.yml] +# │ │ ├── MACHINE_NAME [defined by this file] +# │ │ ├── AUTOTEST_COMMIT [defined by .gitlab-ci.yml] +# │ │ ├── AUTOTEST_ROOT [defined by .gitlab-ci.yml] +# │ │ ├── BASELINE_TEST [defined by this file] +# │ │ └── REBASELINE [defined by .gitlab-ci.yml] +# │ └── subscribed-pipelines.yml +# │ ├── +# │ │ ├── .machine-check +# │ │ ├── generate-job-lists +# │ │ ├── dane-up-check +# │ │ ├── dane-build-and-test +# │ │ ├── dane-baseline +# │ │ ├── tioga-up-check +# │ │ ├── tioga-build-and-test +# │ │ ├── matrix-up-check +# │ │ └── matrix-build-and-test +# │ ├── +# │ │ └── .build-and-test [from ".gitlab-ci.yml"] +# │ ├── +# │ │ └── .gitlab/dane-baseline.yml [by "dane-baseline"] +# │ └── +# │ ├── GITHUB_PROJECT_ORG +# │ ├── GITHUB_PROJECT_NAME +# │ ├── AUTOTEST +# │ ├── AUTOTEST_COMMIT +# │ └── REBASELINE +# └── tests +# ├── gitlab +# │ ├── build_and_test +# │ │ ├── +# │ │ ├── +# │ │ │ ├── tests/uberenv/uberenv.py [deps mode, cloned] +# │ │ │ └── tests/gitlab/get_mfem_uberenv [deps mode] +# │ │ └── +# │ │ ├── SYS_TYPE +# │ │ ├── THREADS [num. parallel jobs to build MFEM] +# │ │ ├── MODULE_LIST [modules to load] +# │ │ ├── CI_JOB_ID +# │ │ ├── USE_DEV_SHM +# │ │ ├── SPACK_DEBUG +# │ │ ├── DEBUG_MODE +# │ │ ├── REGISTRY_TOKEN +# │ │ ├── CI_REGISTRY_USER (defined by Gitlab) +# │ │ ├── USER +# │ │ ├── CI_REGISTRY_IMAGE (defined by Gitlab) +# │ │ └── CI_JOB_TOKEN (defined by Gitlab) +# │ ├── build_and_test_setup +# │ │ ├── +# │ │ └── +# │ │ ├── MFEM_DATA_REPO +# │ │ ├── SHARED_REPOS_DIR +# │ │ ├── AUTOTEST_REPO +# │ │ └── AUTOTEST_ROOT +# │ └── get_mfem_uberenv +# │ ├── tests/uberenv> +# │ └── +# └── uberenv [cloned by tests/gitlab/get_mfem_uberenv] +# └── uberenv.py +# +# +# └── pipelines +# ├── matrix.yml +# │ ├── +# │ │ ├── .on_matrix +# │ │ ├── .matrix_reproducer_init +# │ │ ├── .matrix_reproducer_vars +# │ │ ├── .matrix_reproducer_job +# │ │ ├── .matrix_job_command +# │ │ └── .job_on_matrix +# │ ├── +# │ │ └── .custom_job [from .gitlab/custom-jobs-and-variables.yml] +# │ └── +# │ ├── ON_MATRIX +# │ ├── ADVANCED_JOB +# │ ├── ALL_TARGETS +# │ ├── SYS_TYPE +# │ ├── LLNL_SERVICE_USER +# │ ├── USER +# │ ├── GITHUB_PROJECT_NAME +# │ ├── GITHUB_PROJECT_ORG +# │ ├── MATRIX_JOB_ALLOC +# │ └── JOB_CMD +# ├── dane.yml +# │ ├── +# │ │ ├── .on_dane +# │ │ ├── .dane_reproducer_init +# │ │ ├── .dane_reproducer_vars +# │ │ ├── .dane_reproducer_job +# │ │ ├── .dane_job_command +# │ │ ├── .job_on_dane +# │ │ ├── allocate_resources +# │ │ └── release_resources +# │ ├── +# │ │ └── .custom_job [from .gitlab/custom-jobs-and-variables.yml] +# │ ├── +# │ │ └── export JOBID +# │ └── +# │ ├── ON_DANE +# │ ├── ADVANCED_JOB +# │ ├── ALL_TARGETS +# │ ├── SYS_TYPE +# │ ├── LLNL_SERVICE_USER +# │ ├── USER +# │ ├── GITHUB_PROJECT_NAME +# │ ├── GITHUB_PROJECT_ORG +# │ ├── DANE_JOB_ALLOC +# │ ├── JOB_CMD +# │ ├── JOBID +# │ ├── ALLOC_NAME +# │ └── DANE_SHARED_ALLOC +# └── tioga.yml +# ├── +# │ ├── .on_tioga +# │ ├── .tioga_reproducer_init +# │ ├── .tioga_reproducer_vars +# │ ├── .tioga_reproducer_job +# │ ├── .tioga_job_command +# │ ├── .job_on_tioga +# │ ├── allocate_resources +# │ └── release_resources +# ├── +# │ └── .custom_job [from .gitlab/custom-jobs-and-variables.yml] +# ├── +# │ └── PROXY +# └── +# ├── ON_TIOGA +# ├── ADVANCED_JOB +# ├── ALL_TARGETS +# ├── SYS_TYPE +# ├── LLNL_SERVICE_USER +# ├── USER +# ├── GITHUB_PROJECT_NAME +# ├── GITHUB_PROJECT_ORG +# ├── TIOGA_JOB_ALLOC +# ├── JOB_CMD +# ├── PROXY +# ├── ALLOC_NAME +# └── TIOGA_SHARED_ALLOC +############################################################################### +# We define the following GitLab pipeline variables: variables: +##### LC GITLAB CONFIGURATION CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner" + +##### PROJECT VARIABLES USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}" SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos" AUTOTEST_ROOT: "${SHARED_REPOS_DIR}" - # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and - # used in '.gitlab/configs/-config.yml': MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data" - +# AUTOTEST: enable (ON/YES) or disable (any other value) test reporting. See +# also AUTOTEST_COMMIT. + AUTOTEST: "OFF" +# AUTOTEST_COMMIT: used only when AUTOTEST is set to ON/YES. +# * If AUTOTEST_COMMIT is set to ON/YES, reporting jobs will commit their +# files to the MFEM/autotest repo. +# * If AUTOTEST_COMMIT is NOT set to ON/YES, reporting jobs will NOT commit +# their files to the MFEM/autotest repo. Instead they will just show the +# contents of the report files and remove them. + AUTOTEST_COMMIT: "ON" +# REBASELINE: # Defines the default choice for updating the saved baseline results. By default # the baseline can only be updated from the master branch. This variable offers # the option to manually ask for rebaselining from another branch if necessary. - REBASELINE: "NO" - AUTOTEST: "NO" - # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES. - # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their - # files to the MFEM/autotest repo. - # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their - # files to the MFEM/autotest repo. Instead they will just show the contents - # of the report files and remove them. - AUTOTEST_COMMIT: "YES" + REBASELINE: "OFF" -# Trigger subpipelines: -ruby-build-and-test: - stage: sub-pipelines +##### SHARED_CI CONFIGURATION +# Required information about GitHub repository + GITHUB_PROJECT_NAME: "mfem" + GITHUB_PROJECT_ORG: "MFEM" +# Override the pattern describing branches that will skip the "draft PR filter +# test". Add protected branches here. See default value in +# preliminary-ignore-draft-pr.yml. +# ALWAYS_RUN_PATTERN: "" + +############################################################################### +##### High level stages +# We organize the test-pipelines stage with sub-pipelines. Each sub-pipeline +# corresponds to a test batch on a given machine. +stages: + - prerequisites + - test-pipelines + +############################################################################### +# Template for jobs triggering a build-and-test sub-pipeline: +.build-and-test: + stage: test-pipelines variables: - # Explicitly pass down values that we want to be able to set when triggering - # pipelines manually or using scheduling + # Explicitly pass down values that are not always propagated to child + # pipelines, e.g. when a variable is set in the "Settings -> CI" web + # interface (project variables). + # Note: in some cases, this does not work as expected, e.g. when the + # variable is not re-defined in the web interface; in such cases, the child + # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if + # AUTOTEST is undefined, even though there is a default value in + # .gitlab-ci.yml. AUTOTEST: "${AUTOTEST}" AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}" trigger: - include: .gitlab/ruby-build-and-test.yml + include: + - local: '.gitlab/custom-jobs-and-variables.yml' + - project: 'radiuss/radiuss-shared-ci' + ref: 'v2025.09.1' + file: 'pipelines/${CI_MACHINE}.yml' + - artifact: '${CI_MACHINE}-jobs.yml' + job: 'generate-job-lists' strategy: depend + forward: + pipeline_variables: true -ruby-baseline: - stage: sub-pipelines - variables: - # Explicitly pass down values that we want to be able to set when triggering - # pipelines manually or using scheduling - REBASELINE: "${REBASELINE}" - AUTOTEST: "${AUTOTEST}" - AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}" - trigger: - include: .gitlab/ruby-baseline.yml - strategy: depend - -lassen-build-and-test: - stage: sub-pipelines - variables: - # Explicitly pass down values that we want to be able to set when triggering - # pipelines manually or using scheduling - AUTOTEST: "${AUTOTEST}" - AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}" - trigger: - include: .gitlab/lassen-build-and-test.yml - strategy: depend - -corona-build-and-test: - stage: sub-pipelines - variables: - # Explicitly pass down values that we want to be able to set when triggering - # pipelines manually or using scheduling - AUTOTEST: "${AUTOTEST}" - AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}" - trigger: - include: .gitlab/corona-build-and-test.yml - strategy: depend +############################################################################### +include: + # Sets ID tokens for every job using `default:` + - project: 'lc-templates/id_tokens' + file: 'id_tokens.yml' + # [Optional] checks preliminary to running the actual CI test + #- project: 'radiuss/radiuss-shared-ci' + # ref: 'v2025.09.1' + # file: 'preliminary-ignore-draft-pr.yml' + # pipelines subscribed by the project + - local: '.gitlab/subscribed-pipelines.yml' diff --git a/.gitlab/README.md b/.gitlab/README.md index 3f5b76aa30..3e05446b87 100644 --- a/.gitlab/README.md +++ b/.gitlab/README.md @@ -8,6 +8,8 @@ https://mfem.org +FIXME: this file needs to be updated + This directory contains most of the GitLab CI configuration. MFEM runs both PR and nightly testing on GitLab. @@ -15,18 +17,18 @@ and nightly testing on GitLab. ## Top level -The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo. -This file only defines one stage, in which we trigger several -sub-pipelines. +The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo. This +file only defines three stages, a prerequisites one, and two main stages in +which we trigger several sub-pipelines. We use sub-pipelines to isolate the test for one combination of `machine` and `test type`. Machines typically include: -* Ruby: 2nd Gen Intel Xeon (Cascade Lake) -* Lassen: Power9 + Nvidia GPU -* Corona: AMD GPU +* Dane: Intel Sapphire Rapids +* Matrix: Intel Sapphire Rapids + Nvidia H100 GPU +* Tioga: AMD MI250X GPU Test types include: @@ -39,9 +41,31 @@ altering the scheduling, execution and displaying of the others. ## Sub-pipelines -Each file is this directory is the root configuration file for one -sub-pipeline. The naming reflects the corresponding couple (`machine`, -`test_type`). +### build-and-test + +The build-and-test sub-pipelines leverage RADIUSS Shared CI to share most of +the CI implementation. RADIUSS Shared CI provides a shared CI infrastructure +vetted on most LC systems of interest and efficiently leveraging each machine +scheduler to increase CI throughput. The maintenance of RADIUSS Shared CI is +shared among several RADIUSS projects. + +Jobs for the build-and-test sub-pipelines are defined in the jobs directory. +Because build-and-test jobs leverage Uberenv and Spack to build the +dependencies automatically, the jobs essentially consists in a `spack spec` +defined in the jobs files, and some scheduling parameters defined in the +`.gitlab/custom-jobs-and-variables.yml` file. + +Build-and-test jobs all run the `tests/gitlab/build_and_test` script. + +The build-and-test pipelines are controlled by the +`.gitlab/subscribed-pipelines.yml` which defines which machines to run on and +implements additional features like machine availability check, and job list +generation. + +### baseline + +Baseline sub-pipelines are described by files with names reflecting the +machine it runs on, e.g. `dane-baseline`. Those files define the *stages* and the *jobs* for the sub-pipeline. They also contain any configuration that cannot be shared. For the most part @@ -63,11 +87,11 @@ usage function. This should be improved. # More testing -## Adding a new target to a build_and_test pipeline +## Adding a new target to a build-and-test pipeline -`build_and_test` pipelines rely on Spack to install dependencies. Spack is +`build-and-test` pipelines rely on Spack to install dependencies. Spack is driven by Uberenv which helps freezing Spack configuration: the goal being to -point to specific commit in Spack and isolate its configuration so that it is +point to a specific commit in Spack and isolate its configuration so that it is not influenced by the user environment. More documentation about this can be found in `tests/gitlab`. @@ -76,13 +100,13 @@ with a spack spec of MFEM, within the limits permitted by the MFEM spack package. In any build-and-test sub-pipeline a job basically consists in defining the -spack spec to use. Adding a job on ruby for example resumes to: +spack spec to use. Adding a job on Dane for example resumes to: ```yaml : variables: SPEC: "" - extends: .build_and_test_on_ruby + extends: .job_on_dane ``` The remaining and non trivial work is to make sure this spec is working. To diff --git a/.gitlab/configs/common.yml b/.gitlab/configs/common.yml deleted file mode 100644 index 3ad057e038..0000000000 --- a/.gitlab/configs/common.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -include: - - project: 'lc-templates/id_tokens' - file: 'id_tokens.yml' - -# We define the following GitLab pipeline variables: -variables: - -# The path to the shared resource between all jobs. For example, external -# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built -# once for all targets, so that build happen here. The BUILD_ROOT is unique to -# the pipeline, preventing any form of concurrency with other pipelines. This -# also means that the BUILD_ROOT directory will never be cleaned. -# TODO: add a clean-up mechanism - BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID} - -# On LLNL's ruby, there is only one allocation shared among jobs in order to -# save time and resource. This allocation has to be uniquely named so that we -# are sure to retrieve it. - ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} - -# Git repositories used in the pipeline - TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git - TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git - AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git - MFEM_DATA_REPO: https://github.com/mfem/data.git - -# Directory used to place artifacts. - ARTIFACTS_DIR: artifacts - SLURM_OVERLAP: 1 diff --git a/.gitlab/configs/corona-config.yml b/.gitlab/configs/corona-config.yml deleted file mode 100644 index 6fdd5b3d27..0000000000 --- a/.gitlab/configs/corona-config.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# GitLab pipeline configuration for the Corona machine at LLNL -variables: - MACHINE_NAME: corona - -.on_corona: - tags: - - shell - - corona - rules: - # Don't run corona jobs if... - # Note: This makes corona an "opt-in" machine. To activate builds on corona - # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set - # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a - # pipeline there. - - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"' - when: never - # Don't run autotest update if... - - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"' - when: never - # Report success on success status - - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"' - when: on_success - # Report failure on failure status - - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"' - when: on_failure - # Always release resource - - if: '$CI_JOB_NAME =~ /release_resource/' - when: always - # Always cleanup - - if: '$CI_JOB_NAME =~ /cleanup/' - when: always - # Default is to run if previous stage succeeded - - when: on_success - -# Spack helped builds -# Generic corona build job, extending build script -.build_and_test_on_corona: - extends: [.on_corona] - stage: build_and_test - script: - # THREADS is used by 'tests/gitlab/build_and_test', run below - - export THREADS=12 - - echo ${ALLOC_NAME} - - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) - - echo ${JOBID} - - echo ${MFEM_DATA_DIR} - - echo ${SPEC} - - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data diff --git a/.gitlab/configs/lassen-config.yml b/.gitlab/configs/lassen-config.yml deleted file mode 100644 index 4156f716f8..0000000000 --- a/.gitlab/configs/lassen-config.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# GitLab pipelines configurations for the Lassen machine at LLNL -variables: - MACHINE_NAME: lassen - -.on_lassen: - tags: - - shell - - lassen - rules: - - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ... - when: never - # Don't run autotest update if... - - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"' - when: never - # Report success on success status - - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"' - when: on_success - # Report failure on failure status - - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"' - when: on_failure - # Always cleanup - - if: '$CI_JOB_NAME =~ /cleanup/' - when: always - - when: on_success - -# Lassen uses a different job scheduler (spectrum lsf) that does not allow -# pre-allocation the same way slurm does. We use the pci queue on lassen -# to speed-up the allocation. -.build_and_test_on_lassen: - extends: [.on_lassen] - stage: build_and_test - script: - - echo ${MFEM_DATA_DIR} - - echo ${SPEC} - # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j' - - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data - needs: [setup] diff --git a/.gitlab/configs/report-build-and-test.yml b/.gitlab/configs/report-build-and-test.yml deleted file mode 100644 index 757c3ba04a..0000000000 --- a/.gitlab/configs/report-build-and-test.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# Jobs report -.report_job_success: - script: - - echo ${MACHINE_NAME} - - echo ${AUTOTEST} - - echo ${AUTOTEST_COMMIT} - - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}" - - cd ${AUTOTEST_ROOT} - - | - ( - date - echo "Waiting to acquire lock on '$PWD/autotest.lock' ..." - # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try - # every 5 seconds; we may want to add a counter for the number of - # retries to interrupt a potential infinite loop - while ! flock -n 9; do - sleep 5 - done - echo "Acquired lock on '$PWD/autotest.lock'" - date - # Report SUCCESS while holding the file lock on 'autotest.lock'. - # The next script uses the following environment variables: - # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT - # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL - # It also calls the script '.gitlab/scripts/safe_create_rundir'. - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success - err=$? - # sleep for a period to allow NFS to propagate the above changes; - # clearly, there is no guarantee that other NFS clients will see the - # changes even after the timeout - sleep 10 - exit $err - ) 9> autotest.lock - -.report_job_failure: - script: - - echo ${MACHINE_NAME} - - echo ${AUTOTEST} - - echo ${AUTOTEST_COMMIT} - - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}" - - cd ${AUTOTEST_ROOT} - - | - ( - date - echo "Waiting to acquire lock on '$PWD/autotest.lock' ..." - # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try - # every 5 seconds; we may want to add a counter for the number of - # retries to interrupt a potential infinite loop - while ! flock -n 9; do - sleep 5 - done - echo "Acquired lock on '$PWD/autotest.lock'" - date - # Report FAILURE while holding the file lock on 'autotest.lock'. - # The next script uses the following environment variables: - # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT - # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL - # It also calls the script '.gitlab/scripts/safe_create_rundir'. - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure - err=$? - # sleep for a period to allow NFS to propagate the above changes; - # clearly, there is no guarantee that other NFS clients will see the - # changes even after the timeout - sleep 10 - exit $err - ) 9> autotest.lock diff --git a/.gitlab/configs/ruby-config.yml b/.gitlab/configs/ruby-config.yml deleted file mode 100644 index fffca52bf4..0000000000 --- a/.gitlab/configs/ruby-config.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# GitLab pipelines configurations for the Ruby machine at LLNL -variables: - MACHINE_NAME: ruby - -.on_ruby: - tags: - - shell - - ruby - rules: - # Don't run ruby jobs if... - - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"' - when: never - # Don't run autotest update if... - - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"' - when: never - # Report success on success status - - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"' - when: on_success - # Report failure on failure status - - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"' - when: on_failure - # Always release resource - - if: '$CI_JOB_NAME =~ /release_resource/' - when: always - # Always cleanup - - if: '$CI_JOB_NAME =~ /cleanup/' - when: always - # Default is to run if previous stage succeeded - - when: on_success - -# Spack helped builds -# Generic ruby build job, extending build script -.build_and_test_on_ruby: - extends: [.on_ruby] - stage: build_and_test - script: - # THREADS is used by 'tests/gitlab/build_and_test', run below - - export THREADS=16 - - echo ${ALLOC_NAME} - - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) - - echo ${JOBID} - - echo ${MFEM_DATA_DIR} - - echo ${SPEC} - - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data diff --git a/.gitlab/configs/setup-baseline.yml b/.gitlab/configs/setup-baseline.yml index 7b3cac78dc..8a9ca006b7 100644 --- a/.gitlab/configs/setup-baseline.yml +++ b/.gitlab/configs/setup-baseline.yml @@ -18,7 +18,7 @@ setup_baseline: tags: - shell - - ruby + - dane stage: setup variables: GIT_STRATEGY: none diff --git a/.gitlab/configs/setup-build-and-test.yml b/.gitlab/configs/setup-build-and-test.yml deleted file mode 100644 index 3c2b57c4be..0000000000 --- a/.gitlab/configs/setup-build-and-test.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test -# script then symlinks the repo to the parent directory of the MFEM source -# directory. Unit tests that depend on the mfem/data repo will then detect that -# this directory is present and be enabled. -setup: - tags: - - shell - - ruby - stage: setup - variables: - GIT_STRATEGY: none - script: - # - # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml' - # and '.gitlab/configs/-config.yml' - # - - echo "MACHINE_NAME = ${MACHINE_NAME}" - - echo "AUTOTEST = ${AUTOTEST}" - - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}" - - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}" - - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR} - - command -v flock || echo "Required command 'flock' not found" - - | - ( - date - echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..." - # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the - # try every 5 seconds; we may want to add a counter for the number of - # retries to interrupt a potential infinite loop - while ! flock -n 9; do - sleep 5 - done - echo "Acquired lock on '$PWD/mfem-data.lock'" - date - # clone/update the mfem/data repo while holding the file lock on - # 'mfem-data.lock' - err=0 - if [[ ! -d "mfem-data" ]]; then - git clone ${MFEM_DATA_REPO} "mfem-data" - else - cd "mfem-data" && git pull && cd .. - fi || err=1 - # sleep for a period to allow NFS to propagate the above changes; - # clearly, there is no guarantee that other NFS clients will see the - # changes even after the timeout - sleep 10 - exit $err - ) 9> mfem-data.lock - # - # Setup ${AUTOTEST_ROOT}/autotest: - # - - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}" - - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT} - - | - ( - date - echo "Waiting to acquire lock on '$PWD/autotest.lock' ..." - # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try - # every 5 seconds; we may want to add a counter for the number of - # retries to interrupt a potential infinite loop - while ! flock -n 9; do - sleep 5 - done - echo "Acquired lock on '$PWD/autotest.lock'" - date - # clone/update the autotest repo while holding the file lock on - # 'autotest.lock' - err=0 - if [[ ! -d "autotest" ]]; then - git clone ${AUTOTEST_REPO} - else - cd autotest && git pull && cd .. - fi || err=1 - # sleep for a period to allow NFS to propagate the above changes; - # clearly, there is no guarantee that other NFS clients will see the - # changes even after the timeout - sleep 10 - exit $err - ) 9> autotest.lock diff --git a/.gitlab/corona-build-and-test.yml b/.gitlab/corona-build-and-test.yml deleted file mode 100644 index 4c5cba4436..0000000000 --- a/.gitlab/corona-build-and-test.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -stages: - - setup - - allocate_resource - - build_and_test - - release_resource_and_report - -# Slurm shared allocation -allocate_resource: - variables: - GIT_STRATEGY: none - extends: .on_corona - stage: allocate_resource - script: - - echo ${ALLOC_NAME} - - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME} - timeout: 6h - needs: [setup] - -# Build and test jobs, simply provide a spec -rocm_gcc_8.3.1: - variables: - SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906" - extends: .build_and_test_on_corona - needs: [allocate_resource] - -# Release slurm allocation -release_resource: - variables: - GIT_STRATEGY: none - extends: .on_corona - stage: release_resource_and_report - script: - - echo ${ALLOC_NAME} - - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) - - echo ${JOBID} - - ([[ -n "${JOBID}" ]] && scancel ${JOBID}) - needs: [rocm_gcc_8.3.1] - -# Jobs report -report_job_success: - stage: release_resource_and_report - extends: - - .on_corona - - .report_job_success - -report_job_failure: - stage: release_resource_and_report - extends: - - .on_corona - - .report_job_failure - -include: - - local: .gitlab/configs/common.yml - - local: .gitlab/configs/corona-config.yml - - local: .gitlab/configs/setup-build-and-test.yml - - local: .gitlab/configs/report-build-and-test.yml diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml new file mode 100644 index 0000000000..295707817a --- /dev/null +++ b/.gitlab/custom-jobs-and-variables.yml @@ -0,0 +1,132 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +include: + - project: 'lc-templates/id_tokens' + file: 'id_tokens.yml' + +# We define the following GitLab pipeline variables: +variables: +# Set the build-and-test command. +# Nested variables are allowed and useful to customize the job command. We +# protect variables with quotes so that their value may remain a string even if +# they contain whitespaces. + JOB_CMD: + value: tests/gitlab/build_and_test --spec \"${SPEC}\" --data-dir ${MFEM_DATA_DIR} --data +# The path to the shared resource between all jobs in the 'dane-baseline' +# pipeline. For example, external repositories like 'tests' and 'tpls' are +# cloned here. Also, 'tpls' is built once for all targets, so that build happens +# here. The BUILD_ROOT is unique to the pipeline, preventing any form of +# concurrency with other pipelines. This directory is removed by the 'cleanup' +# stage in the 'dane-baseline' pipeline. + BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${CI_MACHINE}-pipeline-${CI_PIPELINE_ID} + +# On LLNL's dane and tioga, the 'build-and-test' pipelines creates only one +# allocation shared among jobs in the pipeline in order to save time and +# resources. This allocation has to be uniquely named so that we are sure to +# retrieve it and avoid collisions. + ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} + +# Git repositories used in the pipelines: +# - TPLS_REPO and TESTS_REPO are used only by the 'dane-baseline' pipeline +# - AUTOTEST_REPO is used by all pipelines +# - MFEM_DATA_REPO is used only by the 'build-and-test' pipelines + TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git + TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git + AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git + MFEM_DATA_REPO: https://github.com/mfem/data.git + +# Directory used to place artifacts: +# - ARTIFACTS_DIR is only used by the 'dane-baseline' pipeline + ARTIFACTS_DIR: artifacts + SLURM_OVERLAP: 1 + +# Dane +# Arguments for top level allocation + DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1" +# Arguments for job level allocation +# Note: We repeat the reservation, necessary when jobs are manually re-triggered. + DANE_JOB_ALLOC: "--reservation=ci --overlap --nodes=1" + +# Tioga +# Arguments for top level allocation + TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=45m --nodes=1" +# Arguments for job level allocation + TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s" + +# Matrix +# Arguments for top level allocation + MATRIX_SHARED_ALLOC: "-p pdebug --exclusive --time=45 --nodes=1 -G 4" +# Arguments for job level allocation +# Note: We repeat the reservation, necessary when jobs are manually re-triggered. + MATRIX_JOB_ALLOC: "--overlap --nodes=1" + +# Configuration shared by build and test jobs specific to this project. +# Not all configuration can be shared. Here projects can fine tune the +# CI behavior. +# See Umpire for an example (export junit test reports). +.custom_job: + artifacts: + reports: + +# Note: this part is not used by the 'dane-baseline' pipeline. +# FIXME: BUILD_ROOT, TPLS_REPO, TESTS_REPO are not needed here. +# Also, the definition of SHARED_REPOS_DIR is wrong. +.reproducer_vars: + script: + - | + echo -e " + # Variables \n + export SPEC=\"${SPEC//\"/\\\"}\" \n + # Directories \n + export BUILD_ROOT=\"\${working_dir}\" \n + export SHARED_REPOS_DIR=\"\${BUILD_ROOT}/..\" \n + export MFEM_DATA_DIR=\"\${SHARED_REPOS_DIR}/mfem-data\" \n + # Repositories \n + export TPLS_REPO=\"${TPLS_REPO//\"/\\\"}\" \n + export TESTS_REPO=\"${TESTS_REPO//\"/\\\"}\" \n + export AUTOTEST_REPO=\"${AUTOTEST_REPO//\"/\\\"}\" \n + export MFEM_DATA_REPO=\"${MFEM_DATA_REPO//\"/\\\"}\" \n + # Setup directories \n + ./tests/gitlab/build_and_test_setup \n + # Using the CI build cache is optional and requires a token. Set it like so: \n + # export REGISTRY_TOKEN=\"\" \n" + # + +# Jobs report +.report_job_success: + script: + - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test SUCCESS + rules: + - when: on_success + +.report_job_failure: + script: + - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test FAILURE + rules: + - when: on_failure + +# Keep the following for debugging purposes: renaming this job from +# '.show_variables' to 'show_variables' will insert this debug job at the +# beginning of all child pipelines. +.show_variables: + tags: [shell, oslic] + variables: + GIT_STRATEGY: none + stage: .pre + script: + - | + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "AUTOTEST=${AUTOTEST}" + echo "AUTOTEST_COMMIT=${AUTOTEST_COMMIT}" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + # Fail the job on purpose to prevent the rest of the pipeline from running + false diff --git a/.gitlab/ruby-baseline.yml b/.gitlab/dane-baseline.yml similarity index 73% rename from .gitlab/ruby-baseline.yml rename to .gitlab/dane-baseline.yml index 6ed715f585..7230dac511 100644 --- a/.gitlab/ruby-baseline.yml +++ b/.gitlab/dane-baseline.yml @@ -11,6 +11,7 @@ variables: BASELINE_TEST: baseline + MACHINE_NAME: dane stages: - setup @@ -19,8 +20,27 @@ stages: - cleanup - baseline_publish -baselinecheck_mfem_intel_ruby: - extends: [.on_ruby] +.on_dane: + tags: + - shell + - dane + rules: + # Don't run dane jobs if... + - if: '$ON_DANE == "OFF"' + when: never + # Don't run autotest update if... + # Note: in some cases, the content of AUTOTEST can be '${AUTOTEST}', so we + # need to treat that value as the default value of 'OFF'. + - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "ON" && $AUTOTEST != "YES"' + when: never + # Always cleanup + - if: '$CI_JOB_NAME =~ /cleanup/' + when: always + # Default is to run if previous stage succeeded + - when: on_success + +baselinecheck_mfem_intel_dane: + extends: [.on_dane] stage: baseline_check variables: # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location @@ -29,10 +49,13 @@ baselinecheck_mfem_intel_ruby: # .gitlab/configs/setup-baseline.yml. TPLS_DIR: ${BUILD_ROOT}/tpls script: + - echo "AUTOTEST=$AUTOTEST" + - echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT" + - echo "AUTOTEST_ROOT=$AUTOTEST_ROOT" - echo ${BUILD_ROOT} - echo ${TPLS_DIR} - # Used by the tests in MFEM/tests: - - export MFEM_TEST_NP=48 + # Used by the tests in MFEM/tests, dane has 224 threads/node: + - export MFEM_TEST_NP=192 # The next script uses the following environment variables: # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR, # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME @@ -44,7 +67,7 @@ baselinecheck_mfem_intel_ruby: allow_failure: true cleanup: - extends: .on_ruby + extends: .on_dane stage: cleanup variables: GIT_STRATEGY: none @@ -53,7 +76,7 @@ cleanup: - rm -rf "${BUILD_ROOT}" || true report_baseline: - extends: [.on_ruby] + extends: [.on_dane] stage: baseline_report script: - echo ${MACHINE_NAME} @@ -89,7 +112,13 @@ report_baseline: cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html fi msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))" - if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then + # Note: in some cases, the content of AUTOTEST_COMMIT can be + # '${AUTOTEST_COMMIT}', so we need to treat that value as the default + # value of 'ON'. + if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then + AUTOTEST_COMMIT="ON" + fi + if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then git pull && \ git add ${rundir} && \ git commit -m "${msg}" && \ @@ -113,12 +142,12 @@ report_baseline: exit $err ) 9> autotest.lock -baselinepublish_mfem_ruby: - extends: [.on_ruby] +baselinepublish_mfem_dane: + extends: [.on_dane] stage: baseline_publish rules: - # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"' - - if: '$REBASELINE == "YES"' + # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "ON"' + - if: '$REBASELINE == "ON"' when: manual script: - echo ${BUILD_ROOT} @@ -128,6 +157,5 @@ baselinepublish_mfem_ruby: - .gitlab/scripts/rebaseline include: - - local: .gitlab/configs/common.yml - - local: .gitlab/configs/ruby-config.yml + - local: .gitlab/custom-jobs-and-variables.yml - local: .gitlab/configs/setup-baseline.yml diff --git a/.gitlab/jobs/dane-reports.yml b/.gitlab/jobs/dane-reports.yml new file mode 100644 index 0000000000..68de3692ac --- /dev/null +++ b/.gitlab/jobs/dane-reports.yml @@ -0,0 +1,19 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +# Jobs report +report_job_success: + extends: [.on_dane, .report_job_success] + stage: jobs-stage-3 + +report_job_failure: + extends: [.on_dane, .report_job_failure] + stage: jobs-stage-3 diff --git a/.gitlab/jobs/dane.yml b/.gitlab/jobs/dane.yml new file mode 100644 index 0000000000..466bf2280f --- /dev/null +++ b/.gitlab/jobs/dane.yml @@ -0,0 +1,87 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +# Override reproducer section to define MFEM specific variables. +.dane_reproducer_vars: + script: + - !reference [.reproducer_vars, script] + +# TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY +# cannot be "none" anymore). + +# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test +# script then symlinks the repo to the parent directory of the MFEM source +# directory. Unit tests that depend on the mfem/data repo will then detect that +# this directory is present and be enabled. +setup: + extends: .on_dane + stage: jobs-stage-1 + script: + - ./tests/gitlab/build_and_test_setup + + +######################## +# Overridden shared jobs +######################## +# When using shared jobs, we can duplicate them here to override description and +# add necessary changes. +# We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that +# the comparison with the original job is easier. + + +############ +# Extra jobs +############ +# We do not recommend using ${PROJECT__VARIANTS} and +# ${PROJECT__DEPS} in the extra jobs. There is not reason not to fully +# describe the spec here. + +.mfem_job_on_dane: + extends: .job_on_dane + stage: jobs-stage-2 + variables: + # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32 + THREADS: 28 + +debug_ser_gcc_10: + extends: .mfem_job_on_dane + variables: + SPEC: "%gcc@10.3.1 +debug~mpi" + +debug_par_gcc_10: + extends: .mfem_job_on_dane + variables: + SPEC: "%gcc@10.3.1 +debug+mpi" + +opt_ser_gcc_10: + extends: .mfem_job_on_dane + variables: + SPEC: "%gcc@10.3.1 ~mpi" + +opt_par_gcc_10: + extends: .mfem_job_on_dane + variables: + SPEC: "%gcc@10.3.1" + +opt_par_gcc_10_sundials: + extends: .mfem_job_on_dane + variables: + SPEC: "%gcc@10.3.1 +sundials" + +opt_par_gcc_10_petsc: + extends: .mfem_job_on_dane + variables: + SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist" + +opt_par_gcc_10_pumi: + extends: .mfem_job_on_dane + variables: + SPEC: "%gcc@10.3.1 +pumi" diff --git a/.gitlab/jobs/matrix-reports.yml b/.gitlab/jobs/matrix-reports.yml new file mode 100644 index 0000000000..cbd34786b2 --- /dev/null +++ b/.gitlab/jobs/matrix-reports.yml @@ -0,0 +1,19 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +# Jobs report +report_job_success: + extends: [.on_matrix, .report_job_success] + stage: jobs-stage-3 + +report_job_failure: + extends: [.on_matrix, .report_job_failure] + stage: jobs-stage-3 diff --git a/.gitlab/jobs/matrix.yml b/.gitlab/jobs/matrix.yml new file mode 100644 index 0000000000..c21ce76b83 --- /dev/null +++ b/.gitlab/jobs/matrix.yml @@ -0,0 +1,65 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +# Override reproducer section to define UMPIRE specific variables. +.matrix_reproducer_vars: + script: + - !reference [.reproducer_vars, script] + +#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore). + +# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test +# script then symlinks the repo to the parent directory of the MFEM source +# directory. Unit tests that depend on the mfem/data repo will then detect that +# this directory is present and be enabled. +setup: + extends: .on_matrix + stage: jobs-stage-1 + script: + - ./tests/gitlab/build_and_test_setup + + +######################## +# Overridden shared jobs +######################## +# When using shared jobs , we can duplicate them here to override description and add necessary changes. +# We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that +# the comparison with the original job is easier. + + +############ +# Extra jobs +############ +# We do not recommend using ${PROJECT__VARIANTS} and +# ${PROJECT__DEPS} in the extra jobs. There is not reason not to fully +# describe the spec here. + +.mfem_job_on_matrix: + extends: .job_on_matrix + stage: jobs-stage-2 + variables: + # We run 2 jobs on 1 node that has 112 threads + THREADS: 48 + # These modules need to be consistent with the uberenv configurations: + MODULE_LIST: "gcc/10.3.1-magic cuda/12.9.1" + +allocate_resources: + timeout: 4h + +opt_mpi_cuda_gcc: + extends: .mfem_job_on_matrix + variables: + SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90" + +opt_mpi_cuda_hypre_cuda_gcc: + extends: .mfem_job_on_matrix + variables: + SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda" diff --git a/.gitlab/jobs/tioga-reports.yml b/.gitlab/jobs/tioga-reports.yml new file mode 100644 index 0000000000..ef40b7f321 --- /dev/null +++ b/.gitlab/jobs/tioga-reports.yml @@ -0,0 +1,20 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +# Jobs report +report_job_success: + extends: [.on_tioga, .report_job_success] + stage: jobs-stage-3 + +report_job_failure: + extends: [.on_tioga, .report_job_failure] + stage: jobs-stage-3 + diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml new file mode 100644 index 0000000000..7b8a2c1483 --- /dev/null +++ b/.gitlab/jobs/tioga.yml @@ -0,0 +1,70 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +# Override reproducer section to define UMPIRE specific variables. +.tioga_reproducer_vars: + script: + - !reference [.reproducer_vars, script] + +#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore). + +# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test +# script then symlinks the repo to the parent directory of the MFEM source +# directory. Unit tests that depend on the mfem/data repo will then detect that +# this directory is present and be enabled. +setup: + extends: .on_tioga + stage: jobs-stage-1 + script: + - ./tests/gitlab/build_and_test_setup + +######################## +# Overridden shared jobs +######################## +# When using shared jobs , we can duplicate them here to override description and add necessary changes. +# We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that +# the comparison with the original job is easier. + + +############ +# Extra jobs +############ +# We do not recommend using ${PROJECT__VARIANTS} and +# ${PROJECT__DEPS} in the extra jobs. There is not reason not to fully +# describe the spec here. + +# Build and test jobs, simply provide a spec + +#.tioga_job_command: +# script: +# - echo PROXY="${PROXY}" +# - echo TIOGA_JOB_ALLOC="${TIOGA_JOB_ALLOC}" +# - "printf '#!/bin/bash\n%s\n' \"${JOB_CMD}\" > flux_script.sh" +# - cat flux_script.sh +# - ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${TIOGA_JOB_ALLOC} flux_script.sh ) +# - rm -f flux_script.sh + +.mfem_job_on_tioga: + extends: .job_on_tioga + stage: jobs-stage-2 + variables: + # We run 1 job on 1 node that has 64 threads + THREADS: 64 + +opt_mpi_rocm_hypre_rocm: + extends: .mfem_job_on_tioga + variables: + SPEC: "%rocmcc@=6.3.1 +rocm amdgpu_target=gfx90a ^hypre+rocm" + +# cce_16_0_1: +# extends: .mfem_job_on_tioga +# variables: +# SPEC: "%cce@=16.0.1" diff --git a/.gitlab/lassen-build-and-test.yml b/.gitlab/lassen-build-and-test.yml deleted file mode 100644 index 7c9af564b7..0000000000 --- a/.gitlab/lassen-build-and-test.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -stages: - - setup - - build_and_test - - report - -opt_mpi_cuda_gcc: - variables: - SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70" - extends: .build_and_test_on_lassen - -opt_mpi_cuda_hypre_cuda_gcc: - variables: - SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70" - extends: .build_and_test_on_lassen - -# Jobs report -report_job_success: - stage: report - extends: - - .on_lassen - - .report_job_success - -report_job_failure: - stage: report - extends: - - .on_lassen - - .report_job_failure - -include: - - local: .gitlab/configs/common.yml - - local: .gitlab/configs/lassen-config.yml - - local: .gitlab/configs/setup-build-and-test.yml - - local: .gitlab/configs/report-build-and-test.yml diff --git a/.gitlab/ruby-build-and-test.yml b/.gitlab/ruby-build-and-test.yml deleted file mode 100644 index 56fead0976..0000000000 --- a/.gitlab/ruby-build-and-test.yml +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -stages: - - setup - - allocate_resource - - build_and_test - - release_resource_and_report - -# Allocate -allocate_resource: - variables: - GIT_STRATEGY: none - extends: .on_ruby - stage: allocate_resource - script: - - echo ${ALLOC_NAME} - - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME} - timeout: 6h - -# GitLab jobs for the Ruby machine at LLNL -debug_ser_gcc_10: - variables: - SPEC: "%gcc@10.3.1 +debug~mpi" - extends: .build_and_test_on_ruby - -debug_par_gcc_10: - variables: - SPEC: "%gcc@10.3.1 +debug+mpi" - extends: .build_and_test_on_ruby - -opt_ser_gcc_10: - variables: - SPEC: "%gcc@10.3.1 ~mpi" - extends: .build_and_test_on_ruby - -opt_par_gcc_10: - variables: - SPEC: "%gcc@10.3.1" - extends: .build_and_test_on_ruby - -opt_par_gcc_10_sundials: - variables: - SPEC: "%gcc@10.3.1 +sundials" - extends: .build_and_test_on_ruby - -opt_par_gcc_10_petsc: - variables: - SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist" - extends: .build_and_test_on_ruby - -opt_par_gcc_10_pumi: - variables: - SPEC: "%gcc@10.3.1 +pumi" - extends: .build_and_test_on_ruby - -# Release -release_resource: - variables: - GIT_STRATEGY: none - extends: .on_ruby - stage: release_resource_and_report - script: - - echo ${ALLOC_NAME} - - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) - - echo ${JOBID} - - ([[ -n "${JOBID}" ]] && scancel ${JOBID}) - -# Jobs report -report_job_success: - stage: release_resource_and_report - extends: - - .on_ruby - - .report_job_success - -report_job_failure: - stage: release_resource_and_report - extends: - - .on_ruby - - .report_job_failure - -include: - - local: .gitlab/configs/common.yml - - local: .gitlab/configs/ruby-config.yml - - local: .gitlab/configs/setup-build-and-test.yml - - local: .gitlab/configs/report-build-and-test.yml diff --git a/.gitlab/scripts/baseline b/.gitlab/scripts/baseline index e78bdf304d..9c8d44999d 100755 --- a/.gitlab/scripts/baseline +++ b/.gitlab/scripts/baseline @@ -14,7 +14,7 @@ # locals glob_err=${BASELINE_TEST}.err base=${BASELINE_TEST}-${SYS_TYPE} -if [[ "${MACHINE_NAME}" == "ruby" ]]; then +if [[ "${MACHINE_NAME}" == "dane" ]]; then base="${BASELINE_TEST}-${MACHINE_NAME}" fi base_diff=${base}.diff @@ -31,12 +31,10 @@ cd tests mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST} # run -if [[ "${MACHINE_NAME}" == "ruby" ]]; then +if [[ "${MACHINE_NAME}" == "dane" ]]; then salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}" elif [[ ${MACHINE_NAME} == "corona" ]]; then salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}" -elif [[ ${MACHINE_NAME} == "lassen" ]]; then - lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}" else echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME" exit 1 diff --git a/.gitlab/scripts/rebaseline b/.gitlab/scripts/rebaseline index a5eb81d871..e1ec093b75 100755 --- a/.gitlab/scripts/rebaseline +++ b/.gitlab/scripts/rebaseline @@ -11,7 +11,7 @@ # terms of the BSD-3 license. We welcome feedback and contributions, see file # CONTRIBUTING.md for details. -# There will be collision between corona and ruby baselines. +# There will be collision between corona and dane baselines. # Once the corresponding files have been generated, we can switch to machine # specific ref. ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE} @@ -21,7 +21,7 @@ PATCH_FILE=${ARTIFACT_PATH}.patch FULL_FILE=${ARTIFACT_PATH}.out DIFF_FILE=${ARTIFACT_PATH}.diff -# There will be collision between corona and ruby baselines. +# There will be collision between corona and dane baselines. # Once the corresponding files have been generated, we can switch to machine # specific ref. SAVED_NAME=baseline-${SYS_TYPE}.saved diff --git a/.gitlab/scripts/report_build_and_test b/.gitlab/scripts/report_build_and_test new file mode 100755 index 0000000000..c4cd12dfb8 --- /dev/null +++ b/.gitlab/scripts/report_build_and_test @@ -0,0 +1,118 @@ +#!/bin/bash + +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +function info_msg () +{ + echo "[Information:] ${1}" +} + +function error_msg () +{ + echo "[Error:] ${1}" +} + +# Perform a report while holding a lock file to prevent concurrency on +# the destination. +# Usage: +# locked_clone +function locked_report () +{ + if ! command -v flock + then + error_msg "Required command 'flock' not found" + exit 1 + fi + + info_msg "Will report ${1} while holding a lock in ${2}" + + ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..." + # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the + # try every 5 seconds; we may want to add a counter for the number of + # retries to interrupt a potential infinite loop + while ! flock -n 9; do sleep 5; done + date; info_msg "Acquired lock on '${PWD}/${2}.lock'" + + report ${1} + err=$? + + # sleep for a period to allow NFS to propagate the above changes; + # clearly, there is no guarantee that other NFS clients will see the + # changes even after the timeout + sleep 10 + exit $err + ) 9> ${2}.lock +} + +function report () +{ + if [[ "${1}" == "SUCCESS" ]] + then + info_msg "All the ${MACHINE_NAME} jobs passed" + status_msg="The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." + elif [[ "${1}" == "FAILURE" ]] + then + info_msg "At least one failure on ${MACHINE_NAME}" + status_msg="Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." + else + error_msg "Unknown status: ${1} ... aborting" + exit 1 + fi + + cd ${AUTOTEST_ROOT}/autotest || \ + { error_msg "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; } + mkdir -p ${MACHINE_NAME} + + rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}" + rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir) + + printf "%s\n" "${status_msg}" \ + "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err + + msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))" + + if [[ "${1}" == "FAILURE" ]] + then + # Create 'autotest-email.html' to indicate failure: + cp ${rundir}/gitlab.err ${rundir}/autotest-email.html + fi + + # Note: in some cases, the content of AUTOTEST_COMMIT can be + # '${AUTOTEST_COMMIT}', so we need to treat that value as the default + # value of 'ON'. + if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then + AUTOTEST_COMMIT="ON" + fi + if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then + git pull && \ + git add ${rundir} && \ + git commit -m "${msg}" && \ + ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push + else + for file in ${rundir}/*; do + echo "------------------------------" + echo "Content of '$file'" + echo "******************************" + cat $file + echo "******************************" + done + rm -rf ${rundir} || true + fi +} + +export MACHINE_NAME=${CI_MACHINE} +info_msg "MACHINE_NAME is ${MACHINE_NAME}" +info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}" +info_msg "AUTOTEST=$AUTOTEST" +info_msg "AUTOTEST_COMMIT=$AUTOTEST_COMMIT" + +cd ${AUTOTEST_ROOT} && locked_report ${1} autotest diff --git a/.gitlab/scripts/report_build_and_test_failure b/.gitlab/scripts/report_build_and_test_failure deleted file mode 100755 index ba2229ea70..0000000000 --- a/.gitlab/scripts/report_build_and_test_failure +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -echo "Runs if there was at least one failure on ${MACHINE_NAME}" - -cd ${AUTOTEST_ROOT}/autotest || \ - { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; } -mkdir -p ${MACHINE_NAME} - -rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}" -rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir) - -printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \ - "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err - -msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))" - -# Create 'autotest-email.html' to indicate failure: -cp ${rundir}/gitlab.err ${rundir}/autotest-email.html - -if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then - git pull && \ - git add ${rundir} && \ - git commit -m "${msg}" && \ - ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push -else - for file in ${rundir}/*; do - echo "------------------------------" - echo "Content of '$file'" - echo "******************************" - cat $file - echo "******************************" - done - rm -rf ${rundir} || true -fi diff --git a/.gitlab/scripts/report_build_and_test_success b/.gitlab/scripts/report_build_and_test_success deleted file mode 100755 index 6b44afd6b0..0000000000 --- a/.gitlab/scripts/report_build_and_test_success +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -echo "Can only run if all the ${MACHINE_NAME} jobs passed" - -cd ${AUTOTEST_ROOT}/autotest || \ - { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; } -mkdir -p ${MACHINE_NAME} - -rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}" -rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir) - -printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \ - "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out - -msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))" - -if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then - git pull && \ - git add ${rundir} && \ - git commit -m "${msg}" && \ - ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push -else - for file in ${rundir}/*; do - echo "------------------------------" - echo "Content of '$file'" - echo "******************************" - cat $file - echo "******************************" - done - rm -rf ${rundir} || true -fi diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml new file mode 100644 index 0000000000..282e6b2ff8 --- /dev/null +++ b/.gitlab/subscribed-pipelines.yml @@ -0,0 +1,130 @@ +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +# The template job to test whether a machine is up. +# Expects CI_MACHINE defined to machine name. +.machine-check: + stage: prerequisites + tags: [shell, oslic] + variables: + GIT_STRATEGY: none + script: + - | + if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]] + then + echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m" + false && \ + curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ + --header 'Content-Type: application/json' \ + --header "authorization: Bearer ${GITHUB_TOKEN}" \ + --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }" + exit 1 + fi + +### +# Trigger a build-and-test pipeline for a machine. +# Comment the jobs for machines you don’t need. +### + +# One job to generate the job list for all the subpipelines +generate-job-lists: + stage: prerequisites + tags: [shell, oslic] + variables: + LOCAL_JOBS_PATH: ".gitlab/jobs" + script: + - | + echo "AUTOTEST=$AUTOTEST" + echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT" + echo "AUTOTEST_ROOT=$AUTOTEST_ROOT" + - | + cat ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml + if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]] + then + cat ${LOCAL_JOBS_PATH}/dane-reports.yml >> dane-jobs.yml + fi + - | + cat ${LOCAL_JOBS_PATH}/matrix.yml > matrix-jobs.yml + if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]] + then + cat ${LOCAL_JOBS_PATH}/matrix-reports.yml >> matrix-jobs.yml + fi + - | + cat ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml + if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]] + then + cat ${LOCAL_JOBS_PATH}/tioga-reports.yml >> tioga-jobs.yml + fi + artifacts: + paths: + - dane-jobs.yml + - matrix-jobs.yml + - tioga-jobs.yml + + +# DANE +dane-up-check: + variables: + CI_MACHINE: "dane" + extends: [.machine-check] + +dane-build-and-test: + variables: + CI_MACHINE: "dane" + needs: [dane-up-check, generate-job-lists] + extends: [.build-and-test] + +# DANE, MFEM Specific +dane-baseline: + stage: test-pipelines + variables: + # Explicitly pass down values that are not always propagated to child + # pipelines, e.g. when a variable is set in the "Settings -> CI" web + # interface (project variables). + # Note: in some cases, this does not work as expected, e.g. when the + # variable is not re-defined in the web interface; in such cases, the child + # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if + # AUTOTEST is undefined, even though there is a default value in + # .gitlab-ci.yml. + AUTOTEST: "${AUTOTEST}" + AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}" + trigger: + include: .gitlab/dane-baseline.yml + strategy: depend + forward: + pipeline_variables: true + needs: [dane-up-check] + + +# TIOGA +tioga-up-check: + variables: + CI_MACHINE: "tioga" + extends: [.machine-check] + +tioga-build-and-test: + variables: + CI_MACHINE: "tioga" + needs: [tioga-up-check, generate-job-lists] + extends: [.build-and-test] + + +# Matrix +matrix-up-check: + variables: + CI_MACHINE: "matrix" + extends: [.machine-check] + +matrix-build-and-test: + variables: + CI_MACHINE: "matrix" + needs: [matrix-up-check, generate-job-lists] + extends: [.build-and-test] diff --git a/CHANGELOG b/CHANGELOG index e7d64e473c..bba984d222 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -27,9 +27,27 @@ Discretization improvements - In the ParMoonolith integration, added support for variational resampling of H1 vector fields. +- Added support for boundary integration to the hyperbolic framework. In this + regard, new classes `BdrHyperbolicDirichletIntegrator` and + `BoundaryHyperbolicFlowIntegrator` have been introduced for implementation + of weak Dirichlet boundary conditions with a general flux or for the linear + case respectively. + +- Added method to compute piecewise linear bounds on high-order functions on + tensor-product elements. + +- Parallel anisotropic refinement of hexahedral meshes is now supported, + provided that neighboring hexahedra are not refined in conflicting directions. + A new ParMesh method is added to check for such conflicts, before refinement. + Meshing improvements -------------------- +- Introduced NC-patch NURBS meshes, which are conforming element-wise but allow + for nonconforming patch topology. This new mesh format supports element + spacing formulas for refinement, as well as local refinement factors for a + subset of knot vectors. + - Added support for higher order meshes in Mesh::MakeSimplicial and ParMesh::MakeSimplicial. @@ -44,13 +62,19 @@ GPU computing set. This is most often used for setting constant essential boundary conditions. A new function Vector::SetSubVectorHost has been added in cases where host execution is always needed (e.g. when the DOFs array is small). + - Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU threads, assigning one task per thread. + - Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction` update operator. This supports mixed geometry meshes and variable order spaces, and is the default derefinement operator constructed by `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`. The operator requires `FiniteElementSpace::Nonconforming() == true`. +- Added new method: GridFunction::GetGradients, with GPU support, for computing + the gradients of a GridFunction on all elements. +- Added GPU support in GradientGridFunctionCoefficient and + InnerProductCoefficient by implementing their Project methods. New and updated examples and miniapps ------------------------------------- @@ -60,10 +84,22 @@ New and updated examples and miniapps operators as smoothers. These miniapps can be found in `miniapps/diag-smoothers`. -API changes +- Added a new miniapp (meshing/mesh-bounding-boxes) that computes the bounding + boxes for each element of a given mesh, and the bounds on the determinant of + the Jacobian of the transformation. + +- Added a new miniapp (tools/gridfunction-bounds) to compute piecewise linear + bounds on a given high-order grid function. + +- Added a new miniapp (electromagnetics/lorentz) which computes the trajectory + of a charged particle, subject to Lorentz forces, in electrostatic and/or + magnetostatic fields as computed by the volta or tesla miniapps. + +API changes: ----------- - mfem::internal::tensor and mfem::internal::dual have been moved to mfem::future::tensor and mfem::future::dual. + - API addition: in class `Operator`, added virtual functions: `AbsMult`, and `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`. @@ -71,16 +107,25 @@ Miscellaneous ------------- - Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which automatically select between CUDA or HIP. + - The CUDA-specific names used by some of the unit tests like 'cunit_tests' and 'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the MFEM build configuration. + - Added the option to enable GPU-aware MPI in MFEM using the environment variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'. + - Added parallel Address Sanitizer, serial and parallel Undefined Behavior Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu. +- FindPointsGSLIB has a new constructor that accepts the mesh object and + internally calls the Setup() method so that the user does not have to. + The FreeData() method has also been moved to the destructor so the user does + not need to manually free-up the memory if the destructor is called before + MPI_Finalize(). + Version 4.8, released on Apr 9, 2025 ==================================== diff --git a/CMakeLists.txt b/CMakeLists.txt index 8cf827935f..d137e27d97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,33 +133,49 @@ if (MFEM_USE_CUDA) if (NOT CMAKE_CUDA_HOST_COMPILER) set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) endif() - if (CMAKE_VERSION VERSION_LESS 3.18.0) - set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}") - elseif (NOT CMAKE_CUDA_ARCHITECTURES) - string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}") - if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}") - set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}") - else() - message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}") - endif() + if (NOT CMAKE_CUDA_ARCHITECTURES) + # make CUDA_ARCH resemble the same form as CMAKE_CUDA_ARCHITECTURES + string(REPLACE "sm_" "" CUDA_ARCH_TMP "${CUDA_ARCH}") + string(REPLACE "," ";" CUDA_ARCH "${CUDA_ARCH_TMP}") + set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH}") else() set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}") endif() message(STATUS "Using CUDA architecture: ${CUDA_ARCH}") enable_language(CUDA) if (CMAKE_VERSION VERSION_LESS 3.18.0) - # backup try to detect if this is clang or nvcc - if(CMAKE_CUDA_COMPILER MATCHES "nvcc$") - # nvcc - set(MFEM_CUDA_COMPILER_IS_NVCC ON) - set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr") - endif() + # backup try to detect if this is clang or nvcc + if(CMAKE_CUDA_COMPILER MATCHES "nvcc$") + # nvcc + set(MFEM_CUDA_COMPILER_IS_NVCC ON) + set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr") + if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}" + OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}" + OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}") + set(CUDA_FLAGS "-arch=${CMAKE_CUDA_ARCHITECTURES} ${CUDA_FLAGS}") + else() + # build -gencode sequence for multiple architectures + foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES) + set(CUDA_FLAGS + "-gencode arch=compute_${ENTRY},code=sm_${ENTRY} ${CUDA_FLAGS}") + endforeach() + endif() + else() + # build cuda-gpu-arch sequence for multiple architectures + # does not support all/all-major/native + foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES) + set(CUDA_FLAGS "-cuda-gpu-arch=sm_${ENTRY} ${CUDA_FLAGS}") + endforeach() + endif() else() - if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA") - # nvcc - set(MFEM_CUDA_COMPILER_IS_NVCC ON) - set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr") - endif() + # TODO: all, native, all-major require CMake 3.24+ + # backport support for CMake 3.18 to 3.24 + if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA") + # nvcc + set(MFEM_CUDA_COMPILER_IS_NVCC ON) + set(CUDA_FLAGS + "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr") + endif() endif() set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING "CUDA standard to use.") @@ -242,10 +258,16 @@ endif() # AMD HIP if (MFEM_USE_HIP) - if (HIP_ARCH) - message(STATUS "Using HIP architecture: ${HIP_ARCH}") - set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for") + if (NOT CMAKE_HIP_ARCHITECTURES) + if (HIP_ARCH) + set(CMAKE_HIP_ARCHITECTURES CACHE STRING "HIP targets to compile for" "${HIP_ARCH}") + set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for" FORCE) + endif() + else() + set(HIP_ARCH CACHE STRING "HIP targets to compile for" "${CMAKE_HIP_ARCHITECTURES}") + set(GPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "HIP targets to compile for" FORCE) endif() + message(STATUS "Using HIP architecture: ${CMAKE_HIP_ARCHITECTURES}") if (ROCM_PATH) list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH}) endif() @@ -278,6 +300,22 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP) endif() endif() +# Warn user if deprecated FETCH_TPLS is provided +if (DEFINED FETCH_TPLS) + message(STATUS "Setting MFEM_FETCH_TPLS to user-provided value of FETCH_TPLS (i.e., MFEM_FETCH_TPLS=${FETCH_TPLS})") + set (MFEM_FETCH_TPLS FETCH_TPLS) + message(DEPRECATION "The use of FETCH_TPLS is deprecated and will be removed in future verison. Please use MFEM_FETCH_TPLS instead.") +endif() + + # Umpire (must be included before hypre, so hypre can use it if needed) +if (MFEM_USE_UMPIRE) + # umpire uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27 + if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0) + cmake_policy(SET CMP0146 OLD) + endif() + find_package(UMPIRE REQUIRED) +endif() + # MPI -> hypre; PETSc (optional) if (MFEM_USE_MPI) find_package(MPI REQUIRED) @@ -495,14 +533,13 @@ endif() # RAJA if (MFEM_USE_RAJA) + # RAJA uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27 + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0) + cmake_policy(SET CMP0146 OLD) + endif() find_package(RAJA REQUIRED) endif() -# UMPIRE -if (MFEM_USE_UMPIRE) - find_package(UMPIRE REQUIRED) -endif() - # GOOGLE-BENCHMARK if (MFEM_USE_BENCHMARK) find_package(Benchmark REQUIRED) @@ -596,7 +633,7 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ - ALGOIM ENZYME) + ALGOIM ENZYME CUDA::cudart) # Add all created targets and *_FOUND libraries in the variables TPL_TARGETS and # TPL_LIBRARIES, respectively. @@ -614,6 +651,7 @@ foreach(TPL IN LISTS MFEM_TPLS) endif() endif() endforeach(TPL) + list(REVERSE TPL_LIBRARIES) list(REMOVE_DUPLICATES TPL_LIBRARIES) list(REVERSE TPL_LIBRARIES) diff --git a/INSTALL b/INSTALL index 503ff5d5b7..bd6088ba2b 100644 --- a/INSTALL +++ b/INSTALL @@ -123,7 +123,7 @@ Parallel build: Parallel build with fetching of hypre and METIS: mkdir ; cd - cmake -DMFEM_USE_MPI=YES -DFETCH_TPLS=YES + cmake -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES make -j 4 CUDA build: @@ -668,6 +668,7 @@ The specific libraries and their options are: - OpenMP (optional), usually part of compiler, used when either MFEM_USE_OPENMP or MFEM_USE_LEGACY_OPENMP is set to YES. Options: OPENMP_OPT, OPENMP_LIB. + Versions: OpenMP >= 3.1 when MFEM_USE_OPENMP=YES. - High-resolution POSIX clocks: when using MFEM_TIMER_TYPE = 2, it may be necessary to link with a system library (e.g. librt.so). @@ -847,6 +848,7 @@ The specific libraries and their options are: - HIP (optional), used when MFEM_USE_HIP = YES. URL: https://rocmdocs.amd.com Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB. + Versions: ROCm >= 5.6.1. - OCCA (optional), used when MFEM_USE_OCCA = YES. URL: https://libocca.org @@ -857,7 +859,7 @@ The specific libraries and their options are: URL: https://github.com/CEED/libCEED https://ceed.exascaleproject.org/libceed Options: CEED_DIR, CEED_OPT, CEED_LIB. - Versions: libCEED >= 0.12. + Versions: libCEED >= 0.12.0. - RAJA (optional), used when MFEM_USE_RAJA = YES. Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported. @@ -1079,9 +1081,10 @@ The following options are CMake specific: MFEM_ENABLE_TESTING - Enable the ctest framework for testing. MFEM_ENABLE_EXAMPLES - Build all of the examples by default. MFEM_ENABLE_MINIAPPS - Build all of the miniapps by default. -FETCH_TPLS - Enable fetching of all supported third-party libraries. -HYPRE_FETCH - Enable fetching of hypre. -METIS_FETCH - Enable fetching of metis. +MFEM_FETCH_TPLS - Enable fetching of all supported third-party libraries. +MFEM_FETCH_GSLIB - Enable fetching of gslib. +MFEM_FETCH_HYPRE - Enable fetching of hypre. +MFEM_FETCH_METIS - Enable fetching of metis. External libraries (CMake): --------------------------- @@ -1147,6 +1150,7 @@ The MFEM CMake build system also provides fetching (automated building) for the packages/libraries listed below. Note that when fetching is enabled, any related auto-detection functionality is disabled. + - GSLIB - HYPRE - METIS diff --git a/config/cmake/MFEMConfig.cmake.in b/config/cmake/MFEMConfig.cmake.in index d296b69530..cbe963041d 100644 --- a/config/cmake/MFEMConfig.cmake.in +++ b/config/cmake/MFEMConfig.cmake.in @@ -84,6 +84,31 @@ set_and_check(MFEM_LIBRARY_DIR "@PACKAGE_LIB_INSTALL_DIR@") check_required_components(MFEM) +include(CMakeFindDependencyMacro) + +if (MFEM_USE_CUDA) + # required for projects linking to MFEM+CUDA, even if they don't use CUDA directly + find_dependency(CUDAToolkit) +endif (MFEM_USE_CUDA) + +if (MFEM_USE_HIP) + # hip/rocm uses the modern MFEM way of linking to targets, need to find dependencies + find_dependency(HIP) + find_dependency(HIPBLAS) + find_dependency(HIPSPARSE) + if (MFEM_USE_MPI) + # assume HYPRE uses HIP + # alternatively could check HYPRE_USING_HIP + find_dependency(rocsparse) + find_dependency(rocrand) + find_dependency(rocsolver) + endif (MFEM_USE_MPI) +endif (MFEM_USE_HIP) + +if (MFEM_USE_RAJA) + find_dependency(RAJA) +endif() + if (NOT TARGET mfem) include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake) endif (NOT TARGET mfem) diff --git a/config/cmake/modules/FindGSLIB.cmake b/config/cmake/modules/FindGSLIB.cmake index d56042f16d..b8b3b7b056 100644 --- a/config/cmake/modules/FindGSLIB.cmake +++ b/config/cmake/modules/FindGSLIB.cmake @@ -9,10 +9,47 @@ # terms of the BSD-3 license. We welcome feedback and contributions, see file # CONTRIBUTING.md for details. -# Defines the following variables: +# Defines the following variables if fetching of TPLs is disabled (default): # - GSLIB_FOUND # - GSLIB_LIBRARIES # - GSLIB_INCLUDE_DIRS +# otherwise, the following are defined: +# - GSLIB (imported library target) + +if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS) + enable_language(C) + string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE) + set(GSLIB_FETCH_VERSION 1.0.9) + set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}}) + if (CMAKE_C_FLAGS) + set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}") + endif() + if (BUILD_SHARED_LIBS) + set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC") + endif() + add_library(GSLIB STATIC IMPORTED) + # define external project and create future include directory so it is present + # to pass CMake checks at end of MFEM configuration step + message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}") + set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib) + include(ExternalProject) + ExternalProject_Add(gslib + GIT_REPOSITORY https://github.com/Nek5000/gslib + GIT_TAG v${GSLIB_FETCH_VERSION} + GIT_SHALLOW TRUE + UPDATE_DISCONNECTED TRUE + PREFIX ${PREFIX} + CONFIGURE_COMMAND "" + BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$ "CFLAGS= ${GSLIB_C_FLAGS}" + INSTALL_COMMAND "") + file(MAKE_DIRECTORY ${PREFIX}/include) + # set imported library target properties + add_dependencies(GSLIB gslib) + set_target_properties(GSLIB PROPERTIES + IMPORTED_LOCATION ${PREFIX}/lib/libgs.a + INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include) + return() +endif() include(MfemCmakeUtilities) mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs diff --git a/config/cmake/modules/FindHYPRE.cmake b/config/cmake/modules/FindHYPRE.cmake index 55369c4573..b3b999ed8a 100644 --- a/config/cmake/modules/FindHYPRE.cmake +++ b/config/cmake/modules/FindHYPRE.cmake @@ -27,6 +27,7 @@ if (HYPRE_FOUND OR TARGET HYPRE) if (HYPRE_USING_HIP) find_package(rocsparse REQUIRED) find_package(rocrand REQUIRED) + find_package(rocsolver REQUIRED) endif() if (HYPRE_LIBRARIES AND HYPRE_INCLUDE_DIRS AND HYPRE_VERSION) find_package_handle_standard_args(HYPRE @@ -36,52 +37,92 @@ if (HYPRE_FOUND OR TARGET HYPRE) endif() endif() -if (HYPRE_FETCH OR FETCH_TPLS) +if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS) set(HYPRE_FETCH_VERSION 2.33.0) + set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE") add_library(HYPRE STATIC IMPORTED) # set options and associated dependencies - set(CMAKE_OPTIONS) - list(APPEND CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}) + set(HYPRE_CMAKE_OPTIONS "") + list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}) + # collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL. + get_cmake_property(all_vars VARIABLES) + foreach(var ${all_vars}) + if(var MATCHES "^HYPRE_ENABLE") + list(APPEND HYPRE_CMAKE_OPTIONS "-D${var}:BOOL=${${var}}") + endif() + endforeach() + # process all MFEM_USE variables that impact hypre if (MFEM_USE_CUDA) - list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_CUDA:BOOL=ON) + list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_CUDA:BOOL=ON -DCMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES}) find_package(CUDAToolkit REQUIRED) target_link_libraries(HYPRE INTERFACE CUDA::cusparse CUDA::curand CUDA::cublas) elseif (MFEM_USE_HIP) - list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_HIP:BOOL=ON) + list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_HIP:BOOL=ON) find_package(rocsparse REQUIRED) find_package(rocrand REQUIRED) target_link_libraries(HYPRE INTERFACE rocsparse rocrand) endif() + if (MFEM_USE_CUDA OR MFEM_USE_HIP) + if (MFEM_USE_UMPIRE) + if (EXISTS ${umpire_DIR}) + list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=ON -Dumpire_DIR:PATH=${umpire_DIR}) + else() + message(FATAL_ERROR "MFEM_USE_UMPIRE=ON, however umpire_DIR isn't visible to HYPRE") + endif() + else() + list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=OFF) + message(WARNING +"================================================================================ + Umpire is disabled while building HYPRE with GPU support. + This is not recommended for performance reasons! + Consider enabling Umpire with -DMFEM_USE_UMPIRE=ON and providing -DUMPIRE_DIR. +================================================================================") + endif() + endif() if (MFEM_USE_SINGLE) - list(APPEND CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON) + list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON) endif() # define external project and create future include directory so it is present # to pass CMake checks at end of MFEM configuration step - message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_VERSION} to be built with ${CMAKE_OPTIONS}") - set(PREFIX ${CMAKE_BINARY_DIR}/fetch/hypre) + message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_TAG} to be built with ${HYPRE_CMAKE_OPTIONS}") + set(HYPRE_INSTALL ${CMAKE_BINARY_DIR}/fetch/hypre) include(ExternalProject) ExternalProject_Add(hypre GIT_REPOSITORY https://github.com/hypre-space/hypre.git - GIT_TAG v${HYPRE_FETCH_VERSION} + GIT_TAG ${HYPRE_FETCH_TAG} GIT_SHALLOW TRUE + GIT_PROGRESS TRUE UPDATE_DISCONNECTED TRUE SOURCE_SUBDIR src - PREFIX ${PREFIX} - CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${CMAKE_OPTIONS}) - file(MAKE_DIRECTORY ${PREFIX}/include) + PREFIX ${HYPRE_INSTALL} + BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL} + CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS}) + file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include) # set imported library target properties add_dependencies(HYPRE hypre) set_target_properties(HYPRE PROPERTIES - IMPORTED_LOCATION ${PREFIX}/lib/libHYPRE.a - INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include) + IMPORTED_LOCATION ${HYPRE_INSTALL}/lib/libHYPRE.a + INTERFACE_INCLUDE_DIRECTORIES ${HYPRE_INSTALL}/include) # convert HYPRE version to integer - string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION ${HYPRE_FETCH_VERSION}) - list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION) - list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION) - list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION) - math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}") - # set cache variables that would otherwise be set after mfem_find_package call - set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE) + if (HYPRE_FETCH_TAG MATCHES "^v?([0-9]+)\\.([0-9]+)\\.([0-9]+)$") + # Exact release tag X.Y.Z + string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_TAG}") + elseif (HYPRE_FETCH_VERSION MATCHES "([0-9]+)\\.([0-9]+)(\\.([0-9]+))?") + string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_VERSION}") + else (NOT DEFINED HYPRE_VERSION) + message(FATAL_ERROR "Unable to find HYPRE release version. Please provide it via -DHYPRE_VERSION") + endif() + if (HYPRE_SPLIT_VERSION AND NOT DEFINED HYPRE_VERSION) + list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION) + list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION) + if (HYPRE_SPLIT_VERSION GREATER 2) + list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION) + else() + set(HYPRE_PATCH_VERSION 0) + endif() + math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}") + set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE) + endif() return() endif() @@ -149,7 +190,8 @@ endif() if (HYPRE_FOUND AND HYPRE_USING_HIP) find_package(rocsparse REQUIRED) find_package(rocrand REQUIRED) - list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES}) + find_package(rocsolver REQUIRED) + list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES} roc::rocsolver roc::rocblas) set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING "HYPRE libraries + dependencies." FORCE) message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}") diff --git a/config/cmake/modules/FindMETIS.cmake b/config/cmake/modules/FindMETIS.cmake index 51643d6c58..9bd7949d1a 100644 --- a/config/cmake/modules/FindMETIS.cmake +++ b/config/cmake/modules/FindMETIS.cmake @@ -18,7 +18,7 @@ # - METIS (imported library target) # - METIS_VERSION_5 (cache variable) -if (METIS_FETCH OR FETCH_TPLS) +if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS) set(METIS_FETCH_VERSION 4.0.3) add_library(METIS STATIC IMPORTED) # define external project @@ -32,6 +32,7 @@ if (METIS_FETCH OR FETCH_TPLS) UPDATE_DISCONNECTED TRUE PREFIX ${PREFIX} CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1 + BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/) # set imported library target properties add_dependencies(METIS metis) diff --git a/config/cmake/modules/MfemCmakeUtilities.cmake b/config/cmake/modules/MfemCmakeUtilities.cmake index 1e1b90dd77..36491c33d2 100644 --- a/config/cmake/modules/MfemCmakeUtilities.cmake +++ b/config/cmake/modules/MfemCmakeUtilities.cmake @@ -718,7 +718,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar) get_target_property(IsImported ${tgt} IMPORTED) # message(STATUS "${tgt}[IMPORTED]: ${IsImported}") # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY, - # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE. + # SHARED_LIBRARY, INTERFACE_LIBRARY, UNKNOWN_LIBRARY, EXECUTABLE. get_target_property(type ${tgt} TYPE) # message(STATUS "${tgt}[TYPE]: ${type}") unset(ImportConfig) @@ -766,7 +766,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar) else() message(STATUS " *** Warning: [${tgt}] LOCATION not defined!") endif() - elseif ("${type}" STREQUAL "SHARED_LIBRARY") + elseif ("${type}" STREQUAL "SHARED_LIBRARY" OR "${type}" STREQUAL "UNKNOWN_LIBRARY") get_target_property(Location ${tgt} LOCATION) if (Location) get_filename_component(Dir ${Location} DIRECTORY) @@ -932,12 +932,14 @@ function(mfem_export_mk_files) endif() set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}") set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}") - # For the next 4 variable, these are the values for the build-tree version of + # For the next 4 variables, these are the values for the build-tree version of # 'config.mk' set(MFEM_INC_DIR "${PROJECT_BINARY_DIR}") set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}") set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk") set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}") + # TODO: CUDA/HIP support: + set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}") set(MFEM_MPIEXEC ${MPIEXEC}) if (NOT MFEM_MPIEXEC) set(MFEM_MPIEXEC "mpirun") diff --git a/config/config.hpp b/config/config.hpp index 74d6eff86e..b3b03666b2 100644 --- a/config/config.hpp +++ b/config/config.hpp @@ -23,6 +23,9 @@ #include "_config.hpp" #endif +#include +#include + namespace mfem { diff --git a/config/config.mk.in b/config/config.mk.in index 8d74841ce7..dae3d75606 100644 --- a/config/config.mk.in +++ b/config/config.mk.in @@ -88,6 +88,7 @@ MFEM_BUILD_TAG = @MFEM_BUILD_TAG@ MFEM_PREFIX = @MFEM_PREFIX@ MFEM_INC_DIR = @MFEM_INC_DIR@ MFEM_LIB_DIR = @MFEM_LIB_DIR@ +MFEM_XLINKER = @MFEM_XLINKER@ # Location of test.mk MFEM_TEST_MK = @MFEM_TEST_MK@ diff --git a/config/defaults.cmake b/config/defaults.cmake index bc267ebebe..f31f81736b 100644 --- a/config/defaults.cmake +++ b/config/defaults.cmake @@ -91,9 +91,10 @@ option(MFEM_ENABLE_BENCHMARKS "Build all of the benchmarks" OFF) # Allow a user to specify fetching of certain third-party libraries instead of # searching for existing installations. -option(FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF) -option(HYPRE_FETCH "Enable fetching of hypre" OFF) -option(METIS_FETCH "Enable fetching of METIS" OFF) +option(MFEM_FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF) +option(MFEM_FETCH_GSLIB "Enable fetching of GSLIB" OFF) +option(MFEM_FETCH_HYPRE "Enable fetching of hypre" OFF) +option(MFEM_FETCH_METIS "Enable fetching of METIS" OFF) # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the # autodetected C++ compiler. diff --git a/config/defaults.mk b/config/defaults.mk index 90c8b2e204..08a01c80fd 100644 --- a/config/defaults.mk +++ b/config/defaults.mk @@ -57,7 +57,7 @@ CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \ CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH) # flags for nvcc NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \ - -arch=$(CUDA_ARCH) + -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include" # Prefixes for passing flags to the host compiler and linker when using # CUDA_CXX=nvcc CUDA_XCOMPILER = -Xcompiler= diff --git a/data/mfem.mesh b/data/mfem.mesh new file mode 100644 index 0000000000..e92a2bd022 --- /dev/null +++ b/data/mfem.mesh @@ -0,0 +1,593 @@ +MFEM mesh v1.0 + +# Created by: Pointwise + +# MFEM Geometry Types: +# +# POINT = 0 +# SEGMENT = 1 +# TRIANGLE = 2 +# SQUARE = 3 +# TETRAHEDRON = 4 +# CUBE = 5 +# PRISM = 6 + +dimension +2 + +elements +160 +1 3 1 164 163 0 +1 3 164 165 162 163 +1 3 2 166 164 1 +1 3 166 132 165 164 +1 3 3 167 166 2 +1 3 167 131 132 166 +1 3 4 168 167 3 +1 3 168 130 131 167 +1 3 5 169 168 4 +1 3 169 129 130 168 +1 3 6 170 169 5 +1 3 170 128 129 169 +1 3 171 172 170 6 +1 3 172 127 128 170 +1 3 124 125 172 171 +1 3 125 126 127 172 +1 3 162 165 173 161 +1 3 165 132 133 173 +1 3 161 173 174 160 +1 3 173 133 134 174 +1 3 160 174 175 159 +1 3 174 134 135 175 +1 3 6 7 176 171 +1 3 7 8 177 176 +1 3 171 176 123 124 +1 3 176 177 122 123 +1 3 159 175 178 158 +1 3 175 135 136 178 +1 3 158 178 179 157 +1 3 178 136 137 179 +1 3 157 179 180 156 +1 3 179 137 138 180 +1 3 122 177 181 121 +1 3 177 8 182 181 +1 3 8 9 183 182 +1 3 9 10 184 183 +1 3 10 11 185 184 +1 3 11 12 186 185 +1 3 12 13 187 186 +1 3 13 14 15 187 +1 3 121 181 119 120 +1 3 181 182 118 119 +1 3 182 183 117 118 +1 3 183 184 188 117 +1 3 184 185 109 188 +1 3 185 186 108 109 +1 3 186 187 189 108 +1 3 187 15 16 189 +1 3 109 110 190 188 +1 3 110 111 191 190 +1 3 111 112 113 191 +1 3 188 190 116 117 +1 3 190 191 115 116 +1 3 191 113 114 115 +1 3 189 192 107 108 +1 3 192 193 106 107 +1 3 193 194 105 106 +1 3 194 195 104 105 +1 3 195 196 103 104 +1 3 16 17 192 189 +1 3 17 18 193 192 +1 3 18 19 194 193 +1 3 19 20 195 194 +1 3 20 21 196 195 +1 3 97 98 197 96 +1 3 98 99 198 197 +1 3 99 100 199 198 +1 3 100 101 200 199 +1 3 101 102 201 200 +1 3 102 103 202 201 +1 3 103 196 203 202 +1 3 196 21 22 203 +1 3 96 197 204 95 +1 3 197 198 39 204 +1 3 198 199 38 39 +1 3 199 200 205 38 +1 3 200 201 32 205 +1 3 201 202 31 32 +1 3 202 203 206 31 +1 3 203 22 23 206 +1 3 32 33 207 205 +1 3 33 34 35 207 +1 3 205 207 37 38 +1 3 207 35 36 37 +1 3 39 40 208 204 +1 3 40 41 209 208 +1 3 41 42 210 209 +1 3 42 43 211 210 +1 3 43 44 212 211 +1 3 204 208 94 95 +1 3 208 209 93 94 +1 3 209 210 92 93 +1 3 210 211 91 92 +1 3 211 212 90 91 +1 3 90 212 213 89 +1 3 212 44 214 213 +1 3 44 45 215 214 +1 3 45 46 216 215 +1 3 46 47 217 216 +1 3 47 48 218 217 +1 3 48 49 219 218 +1 3 49 50 51 219 +1 3 89 213 87 88 +1 3 213 214 86 87 +1 3 214 215 85 86 +1 3 215 216 84 85 +1 3 216 217 83 84 +1 3 217 218 82 83 +1 3 218 219 220 82 +1 3 219 51 52 220 +1 3 53 221 220 52 +1 3 221 81 82 220 +1 3 54 222 221 53 +1 3 222 80 81 221 +1 3 55 223 222 54 +1 3 223 79 80 222 +1 3 26 27 224 25 +1 3 27 28 29 224 +1 3 25 224 225 24 +1 3 224 29 30 225 +1 3 24 225 206 23 +1 3 225 30 31 206 +1 3 154 155 226 153 +1 3 155 156 180 226 +1 3 153 226 227 152 +1 3 226 180 138 227 +1 3 152 227 228 151 +1 3 227 138 139 228 +1 3 151 228 229 150 +1 3 228 139 140 229 +1 3 150 229 230 149 +1 3 229 140 141 230 +1 3 149 230 231 148 +1 3 230 141 142 231 +1 3 148 231 232 147 +1 3 231 142 143 232 +1 3 147 232 145 146 +1 3 232 143 144 145 +1 3 56 233 223 55 +1 3 233 78 79 223 +1 3 57 234 233 56 +1 3 234 77 78 233 +1 3 58 235 234 57 +1 3 235 76 77 234 +1 3 61 236 59 60 +1 3 236 235 58 59 +1 3 62 237 236 61 +1 3 237 76 235 236 +1 3 63 238 237 62 +1 3 238 75 76 237 +1 3 64 239 238 63 +1 3 239 74 75 238 +1 3 65 240 239 64 +1 3 240 73 74 239 +1 3 66 241 240 65 +1 3 241 72 73 240 +1 3 67 242 241 66 +1 3 242 71 72 241 +1 3 68 69 242 67 +1 3 69 70 71 242 + +boundary +164 +3 1 0 1 +3 1 1 2 +3 1 2 3 +3 1 3 4 +3 1 4 5 +3 1 5 6 +3 1 6 7 +3 1 7 8 +3 1 8 9 +3 1 9 10 +3 1 10 11 +3 1 11 12 +3 1 12 13 +3 1 13 14 +3 1 16 17 +3 1 17 18 +3 1 18 19 +3 1 19 20 +3 1 20 21 +3 1 21 22 +3 1 22 23 +3 1 23 24 +3 1 24 25 +3 1 25 26 +3 1 26 27 +3 1 27 28 +3 1 28 29 +3 1 29 30 +3 1 30 31 +3 1 31 32 +3 1 32 33 +3 1 33 34 +3 1 34 35 +3 1 35 36 +3 1 36 37 +3 1 37 38 +3 1 38 39 +3 1 39 40 +3 1 40 41 +3 1 41 42 +3 1 42 43 +3 1 43 44 +3 1 49 50 +3 1 48 49 +3 1 47 48 +3 1 46 47 +3 1 45 46 +3 1 44 45 +3 1 52 53 +3 1 53 54 +3 1 54 55 +3 1 57 58 +3 1 56 57 +3 1 55 56 +3 1 60 61 +3 1 61 62 +3 1 62 63 +3 1 63 64 +3 1 64 65 +3 1 65 66 +3 1 66 67 +3 1 67 68 +3 1 75 76 +3 1 74 75 +3 1 73 74 +3 1 72 73 +3 1 71 72 +3 1 70 71 +3 1 76 77 +3 1 77 78 +3 1 78 79 +3 1 81 82 +3 1 80 81 +3 1 79 80 +3 1 82 83 +3 1 83 84 +3 1 84 85 +3 1 85 86 +3 1 86 87 +3 1 87 88 +3 1 94 95 +3 1 93 94 +3 1 92 93 +3 1 91 92 +3 1 90 91 +3 1 96 97 +3 1 95 96 +3 1 97 98 +3 1 98 99 +3 1 99 100 +3 1 100 101 +3 1 101 102 +3 1 102 103 +3 1 107 108 +3 1 106 107 +3 1 105 106 +3 1 104 105 +3 1 103 104 +3 1 108 109 +3 1 109 110 +3 1 110 111 +3 1 111 112 +3 1 112 113 +3 1 113 114 +3 1 114 115 +3 1 115 116 +3 1 116 117 +3 1 119 120 +3 1 118 119 +3 1 117 118 +3 1 131 132 +3 1 130 131 +3 1 129 130 +3 1 128 129 +3 1 127 128 +3 1 126 127 +3 1 132 133 +3 1 133 134 +3 1 134 135 +3 1 137 138 +3 1 136 137 +3 1 135 136 +3 1 138 139 +3 1 139 140 +3 1 140 141 +3 1 141 142 +3 1 142 143 +3 1 143 144 +3 1 147 148 +3 1 146 147 +3 1 153 154 +3 1 152 153 +3 1 151 152 +3 1 150 151 +3 1 149 150 +3 1 148 149 +3 1 156 157 +3 1 157 158 +3 1 158 159 +3 1 161 162 +3 1 160 161 +3 1 159 160 +2 1 69 70 +2 1 68 69 +3 1 88 89 +3 1 89 90 +3 1 121 122 +3 1 120 121 +3 1 123 124 +3 1 122 123 +3 1 125 126 +3 1 124 125 +1 1 144 145 +1 1 145 146 +3 1 15 16 +3 1 14 15 +3 1 50 51 +3 1 51 52 +3 1 59 60 +3 1 58 59 +3 1 154 155 +3 1 155 156 +3 1 163 0 +3 1 162 163 + +vertices +243 +2 +4 4 +4 3.5 +4 3 +4 2.5 +4 2 +4 1.5 +4 1 +4.5 1 +5 1 +5 1.5 +5 2 +5 2.5 +5 3 +5 3.5 +5 4 +5.500 4 +6 4 +6.500 4 +7 4 +7.5 4 +8 4 +8.5 4 +9 4 +9.5 4 +10 4 +10.5 4 +11 4 +11 3.5 +11 3 +10.5 3 +10 3 +9.5 3 +9.5 2.5 +10 2.5 +10.5 2.5 +10.5 2 +10.5 1.5 +10 1.5 +9.5 1.5 +9.5 1 +10 1 +10.5 1 +11 1 +11.5 1 +12 1 +12 1.5 +12 2 +12 2.5 +12 3 +12 3.5 +12 4 +12.5 4 +13 4 +13.333 3.75 +13.666 3.5 +14.000 3.25 +14.333 3.5 +14.666 3.75 +15.000 4 +15.500 4 +16.000 4 +16.000 3.5 +16.000 3 +16.000 2.5 +16.000 2 +16.000 1.5 +16.000 1 +16.000 0.5 +16.000 0 +15.500 0 +15.000 0 +15.000 0.5000000000000002 +15.000 1 +15.000 1.5 +15.000 2 +15.000 2.5 +15.000 3 +14.666 2.75 +14.333 2.5 +14.000 2.25 +13.666 2.5 +13.333 2.75 +13 3 +13 2.5 +13 2 +13 1.5 +13 1 +13 0.500 +13 0 +12.5 0 +12 0 +11.5 0 +11 0 +10.5 0 +10 0 +9.5 0 +9 0 +8.5 0 +8.5 0.5 +8.5 1 +8.5 1.5 +8.5 2 +8.5 2.5 +8.5 3 +8 3 +7.5 3 +7 3 +6.500 3 +6 3 +6 2.5 +6.5 2.5 +7 2.5 +7.5 2.5 +7.5 2 +7.5 1.5 +7.000 1.5 +6.5 1.5 +6 1.5 +6 1 +6 0.5 +6 0 +5.5 0 +5 0 +4.5 0 +4 0 +3.5 0 +3 0 +3 0.500 +3 1 +3 1.5 +3 2 +3 2.5 +3 3 +2.666 2.75 +2.333 2.5 +2.000 2.25 +1.666 2.5 +1.333 2.75 +1.000 3 +1.000 2.5 +1.000 2 +1.000 1.5 +1.000 1 +1.000 0.5000 +1.000 0 +0.5000 0 +0.0000 0 +0.0000 0.5 +0.0000 1 +0.0000 1.5 +0.0000 2 +0.0000 2.5 +0.0000 3 +0.0000 3.5 +0.0000 4 +0.5000 4 +1.000 4 +1.333 3.75 +1.666 3.5 +2.000 3.25 +2.333 3.5 +2.666 3.75 +3 4 +3.5 4 +3.5 3.5 +3 3.5 +3.5 3 +3.5 2.5 +3.5 2 +3.5 1.5 +3.5 1 +4 0.5 +3.5 0.5 +2.666 3.25 +2.333 3 +2.000 2.75 +4.5 0.5 +5 0.5 +1.666 3 +1.333 3.25 +1.000 3.5 +5.5 0.5 +5.500 1 +5.500 1.5 +5.500 2 +5.500 2.5 +5.500 3 +5.500 3.5 +6 2 +6 3.5 +6.5 2 +7 2 +6.5 3.5 +7 3.5 +7.5 3.5 +8 3.5 +8.5 3.5 +9 0.5 +9 1 +9 1.5 +9 2 +9 2.5 +9 3 +9 3.5 +9.5 0.5 +9.5 2 +9.5 3.5 +10 2 +10 0.5 +10.5 0.5 +11 0.5 +11.5 0.5 +12 0.5 +12.5 0.500 +12.5 1 +12.5 1.5 +12.5 2 +12.5 2.5 +12.5 3 +12.5 3.5 +13 3.5 +13.333 3.250 +13.666 3 +14.000 2.75 +10.5 3.5 +10 3.5 +0.500 3.5 +0.500 3 +0.500 2.5 +0.500 2 +0.500 1.5 +0.500 1 +0.500 0.5 +14.333 3 +14.666 3.25 +15.000 3.5 +15.500 3.5 +15.500 3 +15.500 2.5 +15.500 2 +15.500 1.5 +15.500 1 +15.500 0.5 diff --git a/data/nc-nurbs3d.mesh b/data/nc-nurbs3d.mesh new file mode 100644 index 0000000000..153ed80f65 --- /dev/null +++ b/data/nc-nurbs3d.mesh @@ -0,0 +1,342 @@ +MFEM NURBS NC-patch mesh v1.0 +dimension +3 + +elements +13 +0 1 5 0 8 10 11 9 4 6 7 5 +0 1 5 0 18 8 24 32 30 23 36 38 +0 1 5 0 0 18 32 14 12 30 38 29 +0 1 5 0 32 24 10 20 38 36 26 35 +0 1 5 0 14 32 20 2 29 38 35 16 +0 1 5 0 30 23 36 38 31 22 37 39 +0 1 5 0 12 30 38 29 13 31 39 28 +0 1 5 0 38 36 26 35 39 37 27 34 +0 1 5 0 29 38 35 16 28 39 34 17 +0 1 5 0 31 22 37 39 19 9 25 33 +0 1 5 0 13 31 39 28 1 19 33 15 +0 1 5 0 39 37 27 34 33 25 11 21 +0 1 5 0 28 39 34 17 15 33 21 3 + +boundary +31 +9999 3 8 10 6 4 +9999 3 10 11 7 6 +9999 3 11 9 5 7 +9999 3 9 8 4 5 +9999 3 4 6 7 5 +9999 3 32 24 8 18 +9999 3 18 8 23 30 +9999 3 14 32 18 0 +9999 3 0 18 30 12 +9999 3 14 0 12 29 +9999 3 20 10 24 32 +9999 3 10 20 35 26 +9999 3 2 20 32 14 +9999 3 20 2 16 35 +9999 3 2 14 29 16 +9999 3 30 23 22 31 +9999 3 12 30 31 13 +9999 3 29 12 13 28 +9999 3 26 35 34 27 +9999 3 35 16 17 34 +9999 3 16 29 28 17 +9999 3 31 22 9 19 +9999 3 19 9 25 33 +9999 3 13 31 19 1 +9999 3 28 13 1 15 +9999 3 1 19 33 15 +9999 3 27 34 21 11 +9999 3 33 25 11 21 +9999 3 34 17 3 21 +9999 3 17 28 15 3 +9999 3 15 33 21 3 + +vertex_to_knotspan +8 +23 0 1 8 10 11 9 +22 0 2 8 10 11 9 +24 1 0 8 10 11 9 +36 1 1 8 10 11 9 +37 1 2 8 10 11 9 +25 1 3 8 10 11 9 +26 2 1 8 10 11 9 +27 2 2 8 10 11 9 + +coordinates +40 +3 +0 0 0 +0 1 0 +4 0 0 +4 1 0 +0 0 4 +0 1 4 +4 0 4 +4 1 4 +0 0 2 +0 1 2 +4 0 2 +4 1 2 +0 0.333333333333333 0 +0 0.666666666666667 0 +2 0 0 +2 1 0 +4 0.333333333333334 0 +4 0.666666666666667 0 +0 0 1 +0 1 1 +4 0 1 +4 1 1 +0 0.666666666666667 2 +0 0.333333333333333 2 +2 0 2 +2 1 2 +4 0.333333333333333 2 +4 0.666666666666667 2 +2 0.666666666666667 0 +2 0.333333333333333 0 +0 0.333333333333333 1 +0 0.666666666666667 1 +1.81325211007895 0 1 +1.81325211007895 1 1 +4 0.666666666666667 1 +4 0.333333333333333 1 +2 0.333333333333333 2 +2 0.666666666666667 2 +1.81325211007895 0.333333333333333 1 +1.81325211007895 0.666666666666667 1 + +edges +87 +0 8 10 +1 10 11 +0 9 11 +1 8 9 +0 4 6 +1 6 7 +0 5 7 +1 4 5 +2 4 8 +2 6 10 +2 7 11 +2 5 9 +9 18 8 +7 8 24 +9 32 24 +7 18 32 +9 30 23 +7 23 36 +9 38 36 +7 30 38 +3 18 30 +3 8 23 +3 24 36 +3 32 38 +8 0 18 +8 14 32 +7 0 14 +8 12 30 +8 29 38 +7 12 29 +3 0 12 +3 14 29 +6 24 10 +9 20 10 +6 32 20 +6 36 26 +9 35 26 +6 38 35 +3 10 26 +3 20 35 +8 2 20 +6 14 2 +8 16 35 +6 29 16 +3 2 16 +9 31 22 +7 22 37 +9 39 37 +7 31 39 +4 30 31 +4 23 22 +4 36 37 +4 38 39 +8 13 31 +8 28 39 +7 13 28 +4 12 13 +4 29 28 +6 37 27 +9 34 27 +6 39 34 +4 26 27 +4 35 34 +8 17 34 +6 28 17 +4 16 17 +9 19 9 +7 9 25 +9 33 25 +7 19 33 +5 31 19 +5 22 9 +5 37 25 +5 39 33 +8 1 19 +8 15 33 +7 1 15 +5 13 1 +5 28 15 +6 25 11 +9 21 11 +6 33 21 +5 27 11 +5 34 21 +8 3 21 +6 15 3 +5 17 3 + +knotvectors +10 +1 3 0 0 0.5 1 1 +1 4 0 0 0.333333333333333 0.666666666666667 1 1 +1 3 0 0 0.5 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 + +spacing +0 + +weights +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 + +FiniteElementSpace +FiniteElementCollection: NURBS1 +VDim: 3 +Ordering: 1 + +0 0 0 +0 1 0 +4 0 0 +4 1 0 +0 0 4 +0 1 4 +4 0 4 +4 1 4 +0 0 2 +0 1 2 +4 0 2 +4 1 2 +0 0.333333333333333 0 +0 0.666666666666667 0 +2 0 0 +2 1 0 +4 0.333333333333334 0 +4 0.666666666666667 0 +0 0 1 +0 1 1 +4 0 1 +4 1 1 +0 0.666666666666667 2 +0 0.333333333333333 2 +2 0 2 +2 1 2 +4 0.333333333333333 2 +4 0.666666666666667 2 +2 0.666666666666667 0 +2 0.333333333333333 0 +0 0.333333333333333 1 +0 0.666666666666667 1 +1.81325211007895 0 1 +1.81325211007895 1 1 +4 0.666666666666667 1 +4 0.333333333333333 1 +2 0.333333333333333 2 +2 0.666666666666667 2 +1.81325211007895 0.333333333333333 1 +1.81325211007895 0.666666666666667 1 +2 0 4 +4 0.333333333333333 4 +4 0.666666666666667 4 +2 1 4 +0 0.333333333333333 4 +0 0.666666666666667 4 +0 0 3 +4 0 3 +4 1 3 +0 1 3 +2 0 3 +4 0.333333333333333 3 +4 0.666666666666667 3 +2 1 3 +0 0.666666666666667 3 +0 0.333333333333333 3 +2 0.333333333333333 4 +2 0.666666666666667 4 +2 0.333333333333333 3 +2 0.666666666666667 3 diff --git a/data/nc3-nurbs.mesh b/data/nc3-nurbs.mesh new file mode 100644 index 0000000000..e56e01e07e --- /dev/null +++ b/data/nc3-nurbs.mesh @@ -0,0 +1,96 @@ +MFEM NURBS NC-patch mesh v1.0 +dimension +2 + +# rank attr geom ref_type nodes/children +elements +3 +0 1 3 0 0 4 5 1 +0 1 3 0 6 7 4 2 +0 1 3 0 6 3 5 7 + +# attr geom nodes +boundary +7 +1 1 0 4 +1 1 5 1 +1 1 1 0 +1 1 2 6 +1 1 6 3 +1 1 4 2 +1 1 5 3 + +vertex_to_knotspan +1 +7 1 4 5 + +# top-level node coordinates +coordinates +8 +2 +0 0 +0 1 +2 0 +2 1 +1 0 +1 1 +2 0.5 +1 0.5 + +edges +11 +0 0 4 +1 4 5 +0 1 5 +1 0 1 +2 6 7 +4 7 4 +2 2 4 +4 6 2 +3 6 3 +2 3 5 +3 7 5 + +knotvectors +5 +1 3 0 0 0.5 1 1 +1 3 0 0 0.5 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 +1 2 0 0 1 1 + +spacing +0 + +weights +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 +1.0 + +FiniteElementSpace +FiniteElementCollection: NURBS1 +VDim: 2 +Ordering: 1 + +0 0 +0 1 +2 0 +2 1 +1 0 +1 1 +2 0.5 +1 0.5 +0.5 0 +0.5 1 +0 0.5 +0.5 0.5 +mfem_mesh_end diff --git a/doc/CodeDocumentation.dox b/doc/CodeDocumentation.dox index e3c99cac4f..a3ed8d6478 100644 --- a/doc/CodeDocumentation.dox +++ b/doc/CodeDocumentation.dox @@ -202,6 +202,7 @@ namespace mfem { * - Tesla: simple magnetostatics simulation code * - Maxwell: simple transient full-wave electromagnetics simulation code * - Joule: transient magnetics and Joule heating miniapp + * - Lorentz: simple particle tracking code based on the Lorentz force * - Navier: solve the transient incompressible Navier-Stokes equations * - Mobius Strip: generate various Mobius strip-like meshes * - Klein Bottle: generate three types of Klein bottle surfaces diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 016c7c66f8..afa3758184 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -205,6 +205,15 @@ if (MFEM_ENABLE_TESTING) $ "-no-vis" "--mumps-solver" ${MPIEXEC_POSTFLAGS}) endif() + + # Parallel libCEED example + if (MFEM_USE_CEED AND MFEM_USE_MPI) + add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP} + COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP} + ${MPIEXEC_PREFLAGS} + $ "-no-vis" "-d ceed-cpu" "-pa" "-a" + ${MPIEXEC_POSTFLAGS}) + endif() endif() # Include the examples/amgx directory if AmgX is enabled diff --git a/examples/ex1.cpp b/examples/ex1.cpp index 02d8cedc10..c47543b449 100644 --- a/examples/ex1.cpp +++ b/examples/ex1.cpp @@ -27,6 +27,7 @@ // ex1 -m ../data/fichera-amr.mesh // ex1 -m ../data/mobius-strip.mesh // ex1 -m ../data/mobius-strip.mesh -o -1 -sc +// ex1 -m ../data/nc3-nurbs.mesh -o -1 // // Device sample runs: // ex1 -pa -d cuda diff --git a/examples/ex22.cpp b/examples/ex22.cpp index 01ed4b2f70..82c382c064 100644 --- a/examples/ex22.cpp +++ b/examples/ex22.cpp @@ -471,10 +471,13 @@ int main(int argc, char *argv[]) ofstream sol_r_ofs("sol_r.gf"); ofstream sol_i_ofs("sol_i.gf"); + ofstream sol_z_ofs("sol_z.gf"); sol_r_ofs.precision(8); sol_i_ofs.precision(8); + sol_z_ofs.precision(8); u.real().Save(sol_r_ofs); u.imag().Save(sol_i_ofs); + u.Save(sol_z_ofs); } // 14. Send the solution by socket to a GLVis server. diff --git a/examples/ex22p.cpp b/examples/ex22p.cpp index 976502f8cd..8a66e02a96 100644 --- a/examples/ex22p.cpp +++ b/examples/ex22p.cpp @@ -507,10 +507,11 @@ int main(int argc, char *argv[]) // 15. Save the refined mesh and the solution in parallel. This output can be // viewed later using GLVis: "glvis -np -m mesh -g sol". { - ostringstream mesh_name, sol_r_name, sol_i_name; + ostringstream mesh_name, sol_r_name, sol_i_name, sol_z_name; mesh_name << "mesh." << setfill('0') << setw(6) << myid; sol_r_name << "sol_r." << setfill('0') << setw(6) << myid; sol_i_name << "sol_i." << setfill('0') << setw(6) << myid; + sol_z_name << "sol_z." << setfill('0') << setw(6) << myid; ofstream mesh_ofs(mesh_name.str().c_str()); mesh_ofs.precision(8); @@ -518,10 +519,13 @@ int main(int argc, char *argv[]) ofstream sol_r_ofs(sol_r_name.str().c_str()); ofstream sol_i_ofs(sol_i_name.str().c_str()); + ofstream sol_z_ofs(sol_z_name.str().c_str()); sol_r_ofs.precision(8); sol_i_ofs.precision(8); + sol_z_ofs.precision(8); u.real().Save(sol_r_ofs); u.imag().Save(sol_i_ofs); + u.Save(sol_z_ofs); } // 16. Send the solution by socket to a GLVis server. diff --git a/examples/makefile b/examples/makefile index 6342336947..eade53dd44 100644 --- a/examples/makefile +++ b/examples/makefile @@ -173,6 +173,12 @@ ex11p-test-cpardiso: ex11p @$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso) test-par-YES: ex11p-test-cpardiso endif +ifeq ($(MFEM_USE_CEED),YES) +ex1p-test-ceed: ex1p + @$(call mfem-test,$<, $(RUN_MPI),\ + Parallel libCEED example,-d ceed-cpu -pa -a) +test-par-YES: ex1p-test-ceed +endif # Testing: "test" target and mfem-test* variables are defined in config/test.mk @@ -189,8 +195,8 @@ clean-build: clean-exec: @rm -f refined.mesh displaced.mesh mesh.* ex5.mesh ex6p-checkpoint.* @rm -rf Example5* Example9* Example15* Example16* Example23* ParaView - @rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.* order.* - @rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.* + @rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.* sol_z.* + @rm -f order.* ex9.mesh ex9-mesh.* ex9-init.* ex9-final.* @rm -f deformed.* velocity.* elastic_energy.* mode_* mode_deriv_* flux.* @rm -f ex5-p-*.bp ex9-p-*.bp ex12-p-*.bp ex16-p-*.bp @rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.* diff --git a/fem/CMakeLists.txt b/fem/CMakeLists.txt index 03648a974a..83d3ccad9f 100644 --- a/fem/CMakeLists.txt +++ b/fem/CMakeLists.txt @@ -171,12 +171,19 @@ set(HDRS bilinearform.hpp bilinearform_ext.hpp bilininteg.hpp + integ/lininteg_domain_kernels.hpp + integ/bilininteg_dgdiffusion_kernels.hpp + integ/bilininteg_dgtrace_kernels.hpp + integ/bilininteg_vecdiffusion_kernels.hpp + integ/bilininteg_convection_kernels.hpp integ/bilininteg_diffusion_kernels.hpp integ/bilininteg_elasticity_kernels.hpp integ/bilininteg_hcurl_kernels.hpp integ/bilininteg_hdiv_kernels.hpp integ/bilininteg_hcurlhdiv_kernels.hpp integ/bilininteg_mass_kernels.hpp + integ/bilininteg_vecdiffusion_pa.hpp + integ/bilininteg_vecmass_pa.hpp coefficient.hpp complex_fem.hpp convergence.hpp @@ -241,8 +248,13 @@ set(HDRS lor/lor_ams.hpp lor/lor_batched.hpp lor/lor_h1.hpp + lor/lor_dg.hpp lor/lor_nd.hpp lor/lor_rt.hpp + lor/lor_h1_impl.hpp + lor/lor_dg_impl.hpp + lor/lor_nd_impl.hpp + lor/lor_rt_impl.hpp lor/lor_util.hpp multigrid.hpp nonlinearform.hpp diff --git a/fem/bilinearform_ext.cpp b/fem/bilinearform_ext.cpp index 35a8268215..abda80e8a1 100644 --- a/fem/bilinearform_ext.cpp +++ b/fem/bilinearform_ext.cpp @@ -266,11 +266,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m) // Gather the attributes on the host from all the elements const Mesh &mesh = *trial_fes->GetMesh(); - elem_attributes.SetSize(mesh.GetNE()); - for (int i = 0; i < mesh.GetNE(); ++i) - { - elem_attributes[i] = mesh.GetAttribute(i); - } + elem_attributes = &mesh.GetElementAttributes(); } // Construct face restriction operators only if the bilinear form has @@ -329,45 +325,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m) bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height()); } - const Mesh &mesh = *trial_fes->GetMesh(); - // See LinearFormExtension::Update for explanation of f_to_be logic. - std::unordered_map f_to_be; - for (int i = 0; i < mesh.GetNBE(); ++i) - { - const int f = mesh.GetBdrElementFaceIndex(i); - f_to_be[f] = i; - } - const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary); - bdr_attributes.SetSize(nf_bdr); - int f_ind = 0; - int missing_bdr_elems = 0; - for (int f = 0; f < mesh.GetNumFaces(); ++f) - { - if (!mesh.GetFaceInformation(f).IsOfFaceType(FaceType::Boundary)) - { - continue; - } - int attribute = 1; // default value - if (f_to_be.find(f) != f_to_be.end()) - { - const int be = f_to_be[f]; - attribute = mesh.GetBdrAttribute(be); - } - else - { - // If a boundary face does not correspond to the a boundary element, - // we assign it the default attribute of 1. We also generate a - // warning at runtime with the number of such missing elements. - ++missing_bdr_elems; - } - bdr_attributes[f_ind] = attribute; - ++f_ind; - } - if (missing_bdr_elems) - { - MFEM_WARNING("Missing " << missing_bdr_elems << " boundary elements " - "for boundary faces."); - } + bdr_face_attributes = &trial_fes->GetMesh()->GetBdrFaceAttributes(); } } @@ -429,7 +387,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e) { const int attr = d_attr[e]; - if (d_m[attr - 1] == 0) + if (attr <= 0 || d_m[attr - 1] == 0) { for (int i = 0; i < nd; ++i) { @@ -450,7 +408,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const for (int i = 0; i < iSz; ++i) { assemble_diagonal_with_markers(*integrators[i], elem_markers[i], - elem_attributes, localY); + *elem_attributes, localY); } const ElementRestriction* H1elem_restrict = dynamic_cast(elem_restrict); @@ -476,7 +434,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const for (int i = 0; i < iSz; ++i) { assemble_diagonal_with_markers(*integrators[i], elem_markers[i], - elem_attributes, y); + *elem_attributes, y); } } @@ -489,7 +447,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const for (int i = 0; i < n_bdr_integs; ++i) { assemble_diagonal_with_markers(*bdr_integs[i], bdr_markers[i], - bdr_attributes, bdr_face_Y); + *bdr_face_attributes, bdr_face_Y); } bdr_face_restrict_lex->AddAbsMultTranspose(bdr_face_Y, y); } @@ -588,7 +546,7 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y, for (int i = 0; i < iSz; ++i) { AddMultWithMarkers(*integrators[i], localX, elem_markers[i], - elem_attributes, false, localY, useAbs); + *elem_attributes, false, localY, useAbs); } if (H1elem_restrict && useAbs) { @@ -690,8 +648,8 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y, } for (int i = 0; i < n_bdr_integs; ++i) { - AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes, - false, bdr_face_Y); + AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], + *bdr_face_attributes, false, bdr_face_Y); } for (int i = 0; i < n_bdr_face_integs; ++i) { @@ -699,12 +657,14 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y, { AddMultNormalDerivativesWithMarkers( *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn, - bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn); + bdr_face_markers[i], *bdr_face_attributes, bdr_face_Y, + bdr_face_dYdn); } else { - AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i], - bdr_attributes, false, bdr_face_Y); + AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, + bdr_face_markers[i], *bdr_face_attributes, false, + bdr_face_Y); } } bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y); @@ -727,7 +687,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const localY = 0.0; for (int i = 0; i < iSz; ++i) { - AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes, + AddMultWithMarkers(*integrators[i], localX, elem_markers[i], *elem_attributes, true, localY); } elem_restrict->MultTranspose(localY, y); @@ -774,13 +734,14 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const bdr_face_Y = 0.0; for (int i = 0; i < n_bdr_integs; ++i) { - AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes, - true, bdr_face_Y); + AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], + *bdr_face_attributes, true, bdr_face_Y); } for (int i = 0; i < n_bdr_face_integs; ++i) { - AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i], - bdr_attributes, true, bdr_face_Y); + AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, + bdr_face_markers[i], *bdr_face_attributes, true, + bdr_face_Y); } bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y); } @@ -804,7 +765,7 @@ static void AddWithMarkers_( mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e) { const int attr = d_attr[e]; - if (d_m[attr - 1] == 0) { return; } + if (attr <= 0 || d_m[attr - 1] == 0) { return; } for (int i = 0; i < nd; ++i) { d_y(i, e) += d_x(i, e); @@ -920,7 +881,8 @@ void EABilinearFormExtension::Assemble() { const int i = idx % sz; const int e = idx / sz; - const real_t val = d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0.0; + const real_t val = + d_a[e] > 0 ? (d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0) : 0; if (add) { d_ea_2(i, e) += val; @@ -953,7 +915,7 @@ void EABilinearFormExtension::Assemble() ea_data_tmp.SetSize(ea_data.Size()); integrators[i]->AssembleEA(*a->FESpace(), ea_data_tmp, false); add_with_markers(ea_data_tmp, ea_data, ne, *markers, - elem_attributes, add); + *elem_attributes, add); } } } @@ -982,7 +944,7 @@ void EABilinearFormExtension::Assemble() ea_data_tmp.SetSize(ea_data_bdr.Size()); bdr_integs[i]->AssembleEABoundary(*a->FESpace(), ea_data_tmp, add); add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers, - bdr_attributes, add); + *bdr_face_attributes, add); } } } @@ -1031,7 +993,7 @@ void EABilinearFormExtension::Assemble() ea_data_tmp, add); add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers, - bdr_attributes, add); + *bdr_face_attributes, add); } } } diff --git a/fem/bilinearform_ext.hpp b/fem/bilinearform_ext.hpp index cfb11938cd..315f767555 100644 --- a/fem/bilinearform_ext.hpp +++ b/fem/bilinearform_ext.hpp @@ -69,7 +69,8 @@ class PABilinearFormExtension : public BilinearFormExtension protected: const FiniteElementSpace *trial_fes, *test_fes; // Not owned /// Attributes of all mesh elements. - Array elem_attributes, bdr_attributes; + const Array *elem_attributes; // Not owned + const Array *bdr_face_attributes; // Not owned mutable Vector tmp_evec; // Work array mutable Vector localX, localY; mutable Vector int_face_X, int_face_Y; diff --git a/fem/bilininteg.cpp b/fem/bilininteg.cpp index d6872fccc7..36183fcd8d 100644 --- a/fem/bilininteg.cpp +++ b/fem/bilininteg.cpp @@ -3066,7 +3066,6 @@ void VectorDiffusionIntegrator::AssembleElementMatrix( for (int i = 0; i < ir -> GetNPoints(); i++) { - const IntegrationPoint &ip = ir->IntPoint(i); el.CalcDShape(ip, dshape); diff --git a/fem/bilininteg.hpp b/fem/bilininteg.hpp index f710f6c688..94362525a2 100644 --- a/fem/bilininteg.hpp +++ b/fem/bilininteg.hpp @@ -23,6 +23,8 @@ namespace mfem { +class QuadratureSpace; +class FaceQuadratureSpace; /// Abstract base class BilinearFormIntegrator class BilinearFormIntegrator : public NonlinearFormIntegrator @@ -2494,8 +2496,7 @@ private: #endif public: - ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0) - : Q(&q) { alpha = a; } + ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0); void AssembleElementMatrix(const FiniteElement &, ElementTransformation &, @@ -2528,6 +2529,28 @@ public: bool SupportsCeed() const override { return DeviceCanUseCeed(); } + /// arguments: NE, B, G, Bt, Gt, pa_data, x, y, D1D, Q1D + using ApplyKernelType = void (*)(const int, const Array &, + const Array &, + const Array &, + const Array &, const Vector &, + const Vector &, Vector &, const int, + const int); + + /// arguments: DIMS, D1D, Q1D + MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int)); + /// arguments: DIMS, D1D, Q1D + MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int)); + + template + static void AddSpecialization() + { + ApplyPAKernels::Specialization::Add(); + ApplyPATKernels::Specialization::Add(); + } + + struct Kernels { Kernels(); }; + protected: const IntegrationRule* GetDefaultIntegrationRule( const FiniteElement& trial_fe, @@ -2573,41 +2596,40 @@ public: by scalar FE through standard transformation. */ class VectorMassIntegrator: public BilinearFormIntegrator { -private: - int vdim; + int vdim = -1, Q_order = 0; Vector shape, te_shape, vec; DenseMatrix partelmat; DenseMatrix mcoeff; - int Q_order; protected: - Coefficient *Q; - VectorCoefficient *VQ; - MatrixCoefficient *MQ; + Coefficient *Q = nullptr; + VectorCoefficient *VQ = nullptr; + MatrixCoefficient *MQ = nullptr; // PA extension - Vector pa_data; const DofToQuad *maps; ///< Not owned const GeometricFactors *geom; ///< Not owned - int dim, ne, nq, dofs1D, quad1D; + int ne, dim, dofs1D, quad1D, coeff_vdim; + Vector pa_data; public: /// Construct an integrator with coefficient 1.0 - VectorMassIntegrator() - : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { } + VectorMassIntegrator() = default; + /** Construct an integrator with scalar coefficient q. If possible, save memory by using a scalar integrator since the resulting matrix is block diagonal with the same diagonal block repeated. */ - VectorMassIntegrator(Coefficient &q, int qo = 0) - : vdim(-1), Q_order(qo), Q(&q), VQ(NULL), MQ(NULL) { } - VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir) - : BilinearFormIntegrator(ir), vdim(-1), Q_order(0), Q(&q), VQ(NULL), - MQ(NULL) { } + VectorMassIntegrator(Coefficient &q, int qo = 0): Q_order(qo), Q(&q) { } + + VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir): + BilinearFormIntegrator(ir), Q(&q) { } + /// Construct an integrator with diagonal coefficient q - VectorMassIntegrator(VectorCoefficient &q, int qo = 0) - : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(&q), MQ(NULL) { } + VectorMassIntegrator(VectorCoefficient &q, int qo = 0): + vdim(q.GetVDim()), Q_order(qo), VQ(&q) { } + /// Construct an integrator with matrix coefficient q - VectorMassIntegrator(MatrixCoefficient &q, int qo = 0) - : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(NULL), MQ(&q) { } + VectorMassIntegrator(MatrixCoefficient &q, int qo = 0): + vdim(q.GetVDim()), Q_order(qo), MQ(&q) { } int GetVDim() const { return vdim; } void SetVDim(int vdim_) { vdim = vdim_; } @@ -2619,6 +2641,7 @@ public: const FiniteElement &test_fe, ElementTransformation &Trans, DenseMatrix &elmat) override; + using BilinearFormIntegrator::AssemblePA; void AssemblePA(const FiniteElementSpace &fes) override; void AssembleMF(const FiniteElementSpace &fes) override; @@ -2627,6 +2650,15 @@ public: void AddMultPA(const Vector &x, Vector &y) const override; void AddMultMF(const Vector &x, Vector &y) const override; bool SupportsCeed() const override { return DeviceCanUseCeed(); } + + using VectorMassAddMultPAType = + void(*)(const int, const int, + const Array&, const Vector&, + const Vector&, Vector&, const int, const int); + + MFEM_REGISTER_KERNELS(VectorMassAddMultPA, + VectorMassAddMultPAType, + (int, int, int)); }; @@ -2798,15 +2830,13 @@ protected: bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient public: - CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; } + CurlCurlIntegrator(); /// Construct a bilinear form integrator for Nedelec elements - CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) : - BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL) { } + CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr); CurlCurlIntegrator(DiagonalMatrixCoefficient &dq, - const IntegrationRule *ir = NULL) : - BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL) { } - CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) : - BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq) { } + const IntegrationRule *ir = nullptr); + CurlCurlIntegrator(MatrixCoefficient &mq, + const IntegrationRule *ir = nullptr); /* Given a particular Finite Element, compute the element curl-curl matrix elmat */ @@ -2836,6 +2866,34 @@ public: void AssembleDiagonalPA(Vector& diag) override; const Coefficient *GetCoefficient() const { return Q; } + + /// arguments: d1d, q1d, symmetric, NE, bo, bc, bot, bct, gc, gct, pa_data, + /// x, y, useAbs + using ApplyKernelType = void (*)( + const int, const int, const bool, const int, const Array &, + const Array &, const Array &, const Array &, + const Array &, const Array &, const Vector &, + const Vector &, Vector &, const bool); + + /// arguments: d1d, q1d, symmetric, ne, Bo, Bc, Go, Gc, pa_data, diag + using DiagonalKernelType = void (*)(const int, const int, const bool, + const int, const Array &, + const Array &, + const Array &, + const Array &, const Vector &, + Vector &); + + /// parameters: dim, d1d, q1d + MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int)); + /// parameters: dim, d1d, q1d + MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int)); + struct Kernels { Kernels(); }; + + template static void AddSpecialization() + { + ApplyPAKernels::Specialization::Add(); + DiagonalPAKernels::Specialization::Add(); + } }; /** Integrator for $(\mathrm{curl}(u), \mathrm{curl}(v))$ for FE spaces defined by 'dim' copies of a @@ -3071,39 +3129,34 @@ public: to be the spatial dimension (i.e. 2-dimension or 3-dimension). */ class VectorDiffusionIntegrator : public BilinearFormIntegrator { -protected: - Coefficient *Q = NULL; - VectorCoefficient *VQ = NULL; - MatrixCoefficient *MQ = NULL; - - // PA extension - const DofToQuad *maps; ///< Not owned - const GeometricFactors *geom; ///< Not owned - int dim, sdim, ne, dofs1D, quad1D; - Vector pa_data; - -private: - DenseMatrix dshape, dshapedxt, pelmat; int vdim = -1; + DenseMatrix dshape, dshapedxt, pelmat; DenseMatrix mcoeff; Vector vcoeff; +protected: + Coefficient *Q = nullptr; + VectorCoefficient *VQ = nullptr; + MatrixCoefficient *MQ = nullptr; + // PA extension + const DofToQuad *maps; ///< Not owned + const GeometricFactors *geom; ///< Not owned + int ne, dim, sdim, dofs1D, quad1D, coeff_vdim; + Vector pa_data; + public: - VectorDiffusionIntegrator() { } + VectorDiffusionIntegrator(const IntegrationRule *ir = nullptr); /** \brief Integrator with unit coefficient for caller-specified vector dimension. If the vector dimension does not match the true dimension of the space, the resulting element matrix will be mathematically invalid. */ - VectorDiffusionIntegrator(int vector_dimension) - : vdim(vector_dimension) { } + VectorDiffusionIntegrator(int vector_dimension); - VectorDiffusionIntegrator(Coefficient &q) - : Q(&q) { } + VectorDiffusionIntegrator(Coefficient &q); - VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir) - : BilinearFormIntegrator(ir), Q(&q) { } + VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir); /** \brief Integrator with scalar coefficient for caller-specified vector dimension. @@ -3113,8 +3166,7 @@ public: If the vector dimension does not match the true dimension of the space, the resulting element matrix will be mathematically invalid. */ - VectorDiffusionIntegrator(Coefficient &q, int vector_dimension) - : Q(&q), vdim(vector_dimension) { } + VectorDiffusionIntegrator(Coefficient &q, int vector_dimension); /** \brief Integrator with \c VectorCoefficient. The vector dimension of the \c FiniteElementSpace is assumed to be the same as the dimension of the @@ -3125,8 +3177,7 @@ public: If the vector dimension does not match the true dimension of the space, the resulting element matrix will be mathematically invalid. */ - VectorDiffusionIntegrator(VectorCoefficient &vq) - : VQ(&vq), vdim(vq.GetVDim()) { } + VectorDiffusionIntegrator(VectorCoefficient &vq); /** \brief Integrator with \c MatrixCoefficient. The vector dimension of the \c FiniteElementSpace is assumed to be the same as the dimension of the @@ -3137,8 +3188,7 @@ public: If the vector dimension does not match the true dimension of the space, the resulting element matrix will be mathematically invalid. */ - VectorDiffusionIntegrator(MatrixCoefficient& mq) - : MQ(&mq), vdim(mq.GetVDim()) { } + VectorDiffusionIntegrator(MatrixCoefficient& mq); void AssembleElementMatrix(const FiniteElement &el, ElementTransformation &Trans, @@ -3146,6 +3196,7 @@ public: void AssembleElementVector(const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun, Vector &elvect) override; + using BilinearFormIntegrator::AssemblePA; void AssemblePA(const FiniteElementSpace &fes) override; void AssembleMF(const FiniteElementSpace &fes) override; @@ -3154,6 +3205,23 @@ public: void AddMultPA(const Vector &x, Vector &y) const override; void AddMultMF(const Vector &x, Vector &y) const override; bool SupportsCeed() const override { return DeviceCanUseCeed(); } + + /// arguments: ne, coeff_vdim, B, G, pa_data, x, y, d1d, q1d, vdim + using ApplyKernelType = void (*)(const int, const int, + const Array &, const Array &, + const Vector &, const Vector &, Vector &, + const int, const int, const int); + + /// arguments: dim, vdim, d1d, q1d + MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int, int)); + + template + static void AddSpecialization() + { + ApplyPAKernels::Specialization::Add(); + } + + // struct Kernels { Kernels(); }; }; /** Integrator for the linear elasticity form: @@ -3307,8 +3375,8 @@ public: class DGTraceIntegrator : public BilinearFormIntegrator { protected: - Coefficient *rho; - VectorCoefficient *u; + Coefficient *rho = nullptr; + VectorCoefficient *u = nullptr; real_t alpha, beta; // PA extension Vector pa_data; @@ -3321,17 +3389,16 @@ private: Vector tr_shape1, te_shape1, tr_shape2, te_shape2; public: + DGTraceIntegrator(real_t a, real_t b); + /// Construct integrator with $\rho = 1$, $\beta = \alpha/2$. - DGTraceIntegrator(VectorCoefficient &u_, real_t a) - { rho = NULL; u = &u_; alpha = a; beta = 0.5*a; } + DGTraceIntegrator(VectorCoefficient &u_, real_t a); /// Construct integrator with $\rho = 1$. - DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b) - { rho = NULL; u = &u_; alpha = a; beta = b; } + DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b); DGTraceIntegrator(Coefficient &rho_, VectorCoefficient &u_, - real_t a, real_t b) - { rho = &rho_; u = &u_; alpha = a; beta = b; } + real_t a, real_t b); using BilinearFormIntegrator::AssembleFaceMatrix; void AssembleFaceMatrix(const FiniteElement &el1, @@ -3370,6 +3437,26 @@ public: static const IntegrationRule &GetRule(Geometry::Type geom, int order, const ElementTransformation &T); + /// arguments: nf, B, Bt, pa_data, x, y, dofs1D, quad1D + using ApplyKernelType = void (*)(const int, const Array &, + const Array &, const Vector &, + const Vector &, Vector &, const int, + const int); + + /// arguments: DIM, d1d, q1d + MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int)); + /// arguments: DIM, d1d, q1d + MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int)); + + template static void AddSpecialization() + { + ApplyPAKernels::Specialization::Add(); + ApplyPATKernels::Specialization::Add(); + } + + struct Kernels { Kernels(); }; + + private: void SetupPA(const FiniteElementSpace &fes, FaceType type); }; @@ -3416,8 +3503,8 @@ public: class DGDiffusionIntegrator : public BilinearFormIntegrator { protected: - Coefficient *Q; - MatrixCoefficient *MQ; + Coefficient *Q = nullptr; + MatrixCoefficient *MQ = nullptr; real_t sigma, kappa; // these are not thread-safe! @@ -3432,15 +3519,11 @@ protected: IntegrationRules irs{0, Quadrature1D::GaussLobatto}; public: - DGDiffusionIntegrator(const real_t s, const real_t k) - : Q(NULL), MQ(NULL), sigma(s), kappa(k) { } - DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k) - : Q(&q), MQ(NULL), sigma(s), kappa(k) { } - DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k) - : Q(NULL), MQ(&q), sigma(s), kappa(k) { } + DGDiffusionIntegrator(const real_t s, const real_t k); + DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k); + DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k); using BilinearFormIntegrator::AssembleFaceMatrix; - void AssembleFaceMatrix(const FiniteElement &el1, - const FiniteElement &el2, + void AssembleFaceMatrix(const FiniteElement &el1, const FiniteElement &el2, FaceElementTransformations &Trans, DenseMatrix &elmat) override; @@ -3459,6 +3542,28 @@ public: const IntegrationRule &GetRule(int order, Geometry::Type geom); + real_t GetPenaltyParameter() const { return kappa; } + + /// arguments: nf, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D, + /// quad1D + using ApplyKernelType = void (*)(const int, const Array &, + const Array &, + const Array &, + const Array &, const real_t, + const Vector &, const Vector &_, + const Vector &, Vector &, Vector &, + const int, const int); + + /// arguments: DIM, d1d, q1d + MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int)); + + template static void AddSpecialization() + { + ApplyPAKernels::Specialization::Add(); + } + + struct Kernels { Kernels(); }; + private: void SetupPA(const FiniteElementSpace &fes, FaceType type); }; diff --git a/fem/ceed/integrators/convection/convection_qf.h b/fem/ceed/integrators/convection/convection_qf.h index 6a871130c4..3fb634c11a 100644 --- a/fem/ceed/integrators/convection/convection_qf.h +++ b/fem/ceed/integrators/convection/convection_qf.h @@ -8,6 +8,7 @@ // MFEM is free software; you can redistribute it and/or modify it under the // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. +#include /// A structure used to pass additional data to f_build_conv and f_apply_conv struct ConvectionContext { @@ -91,7 +92,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for building quadrature data for a convection operator @@ -167,7 +168,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a conv operator @@ -233,7 +234,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a conv operator @@ -381,7 +382,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q, @@ -525,5 +526,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } diff --git a/fem/ceed/integrators/diffusion/diffusion_qf.h b/fem/ceed/integrators/diffusion/diffusion_qf.h index a11f4948f2..55261930a4 100644 --- a/fem/ceed/integrators/diffusion/diffusion_qf.h +++ b/fem/ceed/integrators/diffusion/diffusion_qf.h @@ -8,7 +8,7 @@ // MFEM is free software; you can redistribute it and/or modify it under the // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. - +#include /// A structure used to pass additional data to f_build_diff and f_apply_diff struct DiffusionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; }; @@ -85,7 +85,7 @@ CEED_QFUNCTION(f_build_diff_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for building quadrature data for a diffusion operator @@ -161,7 +161,7 @@ CEED_QFUNCTION(f_build_diff_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a diff operator @@ -241,7 +241,7 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a diff operator @@ -394,7 +394,7 @@ CEED_QFUNCTION(f_apply_diff_mf_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q, @@ -549,5 +549,5 @@ CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } diff --git a/fem/ceed/integrators/mass/mass_qf.h b/fem/ceed/integrators/mass/mass_qf.h index e7172d023d..b1bf35558a 100644 --- a/fem/ceed/integrators/mass/mass_qf.h +++ b/fem/ceed/integrators/mass/mass_qf.h @@ -8,7 +8,7 @@ // MFEM is free software; you can redistribute it and/or modify it under the // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. - +#include /// A structure used to pass additional data to f_build_diff and f_apply_diff struct MassContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; }; @@ -53,7 +53,7 @@ CEED_QFUNCTION(f_build_mass_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for building quadrature data for a mass operator with a @@ -95,7 +95,7 @@ CEED_QFUNCTION(f_build_mass_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a mass operator @@ -135,7 +135,7 @@ CEED_QFUNCTION(f_apply_mass)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a diff operator @@ -199,7 +199,7 @@ CEED_QFUNCTION(f_apply_mass_mf_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q, @@ -266,5 +266,5 @@ CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } diff --git a/fem/ceed/integrators/nlconvection/nlconvection_qf.h b/fem/ceed/integrators/nlconvection/nlconvection_qf.h index affa201764..3c29a560c9 100644 --- a/fem/ceed/integrators/nlconvection/nlconvection_qf.h +++ b/fem/ceed/integrators/nlconvection/nlconvection_qf.h @@ -8,6 +8,7 @@ // MFEM is free software; you can redistribute it and/or modify it under the // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. +#include /// A structure used to pass additional data to f_build_conv and f_apply_conv struct NLConvectionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; }; @@ -87,7 +88,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for building quadrature data for a convection operator @@ -167,7 +168,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a conv operator @@ -247,7 +248,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } /// libCEED Q-function for applying a conv operator @@ -362,7 +363,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q, @@ -475,5 +476,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q, } break; } - return 0; + return CEED_ERROR_SUCCESS; } diff --git a/fem/ceed/interface/ceed.hpp b/fem/ceed/interface/ceed.hpp index 81f097fdce..97ba0c9d4b 100644 --- a/fem/ceed/interface/ceed.hpp +++ b/fem/ceed/interface/ceed.hpp @@ -18,10 +18,21 @@ #include -#if !CEED_VERSION_GE(0,12,0) +#if !CEED_VERSION_GE(0, 12, 0) #error MFEM requires a libCEED version >= 0.12.0 #endif +#if !CEED_VERSION_GE(0, 13, 0) +#define CeedOperatorCreateComposite(ceed, op) \ + CeedCompositeOperatorCreate((ceed), (op)) +#define CeedOperatorCompositeAddSub(op, sub) \ + CeedCompositeOperatorAddSub((op), (sub)) +#define CeedOperatorCompositeGetNumSub(op, num) \ + CeedCompositeOperatorGetNumSub((op), (num)) +#define CeedOperatorCompositeGetSubList(op, list) \ + CeedCompositeOperatorGetSubList((op), (list)) +#endif + namespace mfem { diff --git a/fem/ceed/interface/mixed_integrator.hpp b/fem/ceed/interface/mixed_integrator.hpp index f8300e36d7..f7296076e9 100644 --- a/fem/ceed/interface/mixed_integrator.hpp +++ b/fem/ceed/interface/mixed_integrator.hpp @@ -83,7 +83,7 @@ public: } // Create composite CeedOperator - CeedCompositeOperatorCreate(internal::ceed, &oper); + CeedOperatorCreateComposite(internal::ceed, &oper); // Create each sub-CeedOperator sub_ops.reserve(element_indices.size()); @@ -101,7 +101,7 @@ public: int nelem = *count[value.first]; sub_op->Assemble(info, fes, ir, nelem, indices, Q); sub_ops.push_back(sub_op); - CeedCompositeOperatorAddSub(oper, sub_op->GetCeedOperator()); + CeedOperatorCompositeAddSub(oper, sub_op->GetCeedOperator()); } const int ndofs = fes.GetVDim() * fes.GetNDofs(); diff --git a/fem/ceed/interface/util.cpp b/fem/ceed/interface/util.cpp index 07ce91e309..75584786af 100644 --- a/fem/ceed/interface/util.cpp +++ b/fem/ceed/interface/util.cpp @@ -140,11 +140,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field) CeedOperator *subops; if (isComposite) { -#if CEED_VERSION_GE(0, 10, 2) - ierr = CeedCompositeOperatorGetSubList(oper, &subops); PCeedChk(ierr); -#else - ierr = CeedOperatorGetSubList(oper, &subops); PCeedChk(ierr); -#endif + ierr = CeedOperatorCompositeGetSubList(oper, &subops); PCeedChk(ierr); ierr = CeedOperatorGetQFunction(subops[0], &qf); PCeedChk(ierr); } else @@ -171,7 +167,11 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field) for (int i = 0; i < numinputfields; ++i) { ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); PCeedChk(ierr); - if (if_vector == CEED_VECTOR_ACTIVE) + bool is_active = if_vector == CEED_VECTOR_ACTIVE; +#if CEED_VERSION_GE(0, 13, 0) + ierr = CeedVectorDestroy(&if_vector); PCeedChk(ierr); +#endif + if (is_active) { if (found) { diff --git a/fem/ceed/solvers/algebraic.cpp b/fem/ceed/solvers/algebraic.cpp index 9b41bd2a33..2c0f86560e 100644 --- a/fem/ceed/solvers/algebraic.cpp +++ b/fem/ceed/solvers/algebraic.cpp @@ -228,7 +228,7 @@ void AddToCompositeOperator(BilinearFormIntegrator *integ, CeedOperator op) { if (integ->SupportsCeed()) { - CeedCompositeOperatorAddSub(op, integ->GetCeedOp().GetCeedOperator()); + CeedOperatorCompositeAddSub(op, integ->GetCeedOp().GetCeedOperator()); } else { @@ -240,7 +240,7 @@ CeedOperator CreateCeedCompositeOperatorFromBilinearForm(BilinearForm &form) { int ierr; CeedOperator op; - ierr = CeedCompositeOperatorCreate(internal::ceed, &op); PCeedChk(ierr); + ierr = CeedOperatorCreateComposite(internal::ceed, &op); PCeedChk(ierr); MFEM_VERIFY(form.GetBBFI()->Size() == 0, "Not implemented for this integrator!"); @@ -271,18 +271,13 @@ CeedOperator CoarsenCeedCompositeOperator( MFEM_ASSERT(isComposite, ""); CeedOperator op_coarse; - ierr = CeedCompositeOperatorCreate(internal::ceed, + ierr = CeedOperatorCreateComposite(internal::ceed, &op_coarse); PCeedChk(ierr); int nsub; CeedOperator *subops; -#if CEED_VERSION_GE(0, 10, 2) - ierr = CeedCompositeOperatorGetNumSub(op, &nsub); PCeedChk(ierr); - ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr); -#else - ierr = CeedOperatorGetNumSub(op, &nsub); PCeedChk(ierr); - ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr); -#endif + ierr = CeedOperatorCompositeGetNumSub(op, &nsub); PCeedChk(ierr); + ierr = CeedOperatorCompositeGetSubList(op, &subops); PCeedChk(ierr); for (int isub=0; isub #include @@ -80,6 +81,49 @@ real_t PWConstCoefficient::Eval(ElementTransformation & T, return (constants(att-1)); } +void PWConstCoefficient::Project(QuadratureFunction &qf) +{ + auto &qs = *qf.GetSpace(); + + const bool compressed = + qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Size() == 1; + const int *offsets = qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Read(); + const int ne = qs.GetNE(); + + const int *attributes = [&]() + { + if (dynamic_cast(&qs) != nullptr) + { + return qs.GetMesh()->GetElementAttributes().Read(); + } + else if (auto *qs_f = dynamic_cast(&qs)) + { + MFEM_VERIFY(qs_f->GetFaceType() == FaceType::Boundary, + "Interior faces do not have attributes."); + return qs.GetMesh()->GetBdrFaceAttributes().Read(); + } + else + { + MFEM_ABORT("Unsupported case."); + } + }(); + + const real_t *d_c = constants.Read(); + real_t *d_qf = qf.Write(); + + mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e) + { + const int a = attributes[e]; + const real_t elementConstant = d_c[a - 1]; + const int begin = compressed ? e*offsets[0] : offsets[e]; + const int end = compressed ? (e+1)*offsets[0] : offsets[e+1]; + for (int i = begin; i < end; ++i) + { + d_qf[i] = elementConstant; + } + }); +} + void PWCoefficient::InitMap(const Array & attr, const Array & coefs) { @@ -519,6 +563,26 @@ void GradientGridFunctionCoefficient::Eval( } } +void GradientGridFunctionCoefficient::Project(QuadratureFunction &qf) +{ + const FiniteElementSpace &fes = *GridFunc->FESpace(); + const Mesh &mesh = *fes.GetMesh(); + const int sdim = mesh.SpaceDimension(); + const int gf_vdim = fes.GetVDim(); // assumed to be 1 in this class + qf.SetVDim(sdim*gf_vdim); + if (mesh.GetNE() == 0) { return; } + // All mesh element must be the same type: + MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension()) == 1, + "All mesh elements must be the same type!"); + const IntegrationRule &ir = qf.GetIntRule(0); + // All elements must use the same quadrature rule: + MFEM_VERIFY(qf.Size() == sdim*gf_vdim*ir.GetNPoints()*mesh.GetNE(), + "All mesh elements must use the same quadrature rule!"); + // QuadratureFunction uses the layout qf_vdim x nq x ne, i.e. + // gf_vdim x sdim x nq x nq, so we need to request QVectorLayout::byVDIM: + GridFunc->GetGradients(ir, qf, QVectorLayout::byVDIM); +} + CurlGridFunctionCoefficient::CurlGridFunctionCoefficient( const GridFunction *gf) : VectorCoefficient(0) @@ -1021,6 +1085,29 @@ void SumCoefficient::SetTime(real_t t) this->Coefficient::SetTime(t); } +void SumCoefficient::Project(QuadratureFunction &qf) +{ + if (a == nullptr) + { + // qf = alpha*aConst + beta * b + const real_t d_alpha_a = aConst*alpha; + const real_t d_beta = beta; + b->Project(qf); + auto d_qf = qf.ReadWrite(); + mfem::forall(qf.Size(), [=] MFEM_HOST_DEVICE (int i) + { + d_qf[i] = d_alpha_a + d_beta*d_qf[i]; + }); + } + else + { + a->Project(qf); + QuadratureFunction qf_b(*qf.GetSpace()); + b->Project(qf_b); + add(alpha, qf, beta, qf_b, qf); + } +} + void ProductCoefficient::SetTime(real_t t) { if (a) { a->SetTime(t); } @@ -1028,6 +1115,23 @@ void ProductCoefficient::SetTime(real_t t) this->Coefficient::SetTime(t); } +void ProductCoefficient::Project(QuadratureFunction &qf) +{ + if (a == nullptr) + { + // qf = aConst * b + b->Project(qf); + qf *= aConst; + } + else + { + a->Project(qf); + QuadratureFunction qf_b(qf.GetSpace()); + b->Project(qf_b); + qf *= qf_b; + } +} + void RatioCoefficient::SetTime(real_t t) { if (a) { a->SetTime(t); } @@ -1035,6 +1139,38 @@ void RatioCoefficient::SetTime(real_t t) this->Coefficient::SetTime(t); } +void RatioCoefficient::Project(QuadratureFunction &qf) +{ + if (b == nullptr) + { + if (a == nullptr) + { + qf = aConst / bConst; + } + else + { + a->Project(qf); + qf *= 1.0/bConst; + } + } + else + { + if (a == nullptr) + { + b->Project(qf); + qf.Reciprocal(); + qf *= aConst; + } + else + { + a->Project(qf); + QuadratureFunction qf_b(qf.GetSpace()); + b->Project(qf_b); + qf /= qf_b; + } + } +} + void PowerCoefficient::SetTime(real_t t) { if (a) { a->SetTime(t); } @@ -1065,6 +1201,41 @@ real_t InnerProductCoefficient::Eval(ElementTransformation &T, return va * vb; } +void InnerProductCoefficient::Project(QuadratureFunction &qf) +{ + MFEM_VERIFY(a->GetVDim() == b->GetVDim(), + "Incompatible vector coefficients: a->GetVDim(): " + << a->GetVDim() << ", b->GetVDim(): " << b->GetVDim()); + + const int vdim = a->GetVDim(); + MFEM_VERIFY(vdim >= 1, "invalid vdim: " << vdim); + + // When running on device, make sure the output data is allocated before any + // local temporary data to reduce potential heap fragmentation: + auto dot_d = qf.Write(); + + QuadratureFunction qf_a(qf.GetSpace(), vdim); + QuadratureFunction qf_b(qf.GetSpace(), vdim); + + a->Project(qf_a); + b->Project(qf_b); + + auto a_d = qf_a.Read(); + auto b_d = qf_b.Read(); + + mfem::forall(qf.GetSpace()->GetSize(), [=] MFEM_HOST_DEVICE (int i) + { + const real_t *ai = a_d + i*vdim; + const real_t *bi = b_d + i*vdim; + real_t dot = ai[0]*bi[0]; + for (int d = 1; d < vdim; d++) + { + dot += ai[d]*bi[d]; + } + dot_d[i] = dot; + }); +} + VectorRotProductCoefficient::VectorRotProductCoefficient(VectorCoefficient &A, VectorCoefficient &B) : a(&A), b(&B), va(A.GetVDim()), vb(B.GetVDim()) diff --git a/fem/coefficient.hpp b/fem/coefficient.hpp index 7bc12d9223..b16c022dff 100644 --- a/fem/coefficient.hpp +++ b/fem/coefficient.hpp @@ -132,6 +132,9 @@ public: /// Evaluate the coefficient. real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override; + + /// Fill the QuadratureFunction @a qf with the piecewise constant values. + void Project(QuadratureFunction &qf) override; }; /** @brief A piecewise coefficient with the pieces keyed off the element @@ -894,6 +897,9 @@ public: void Eval(DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir) override; + /// @copydoc VectorCoefficient::Project(QuadratureFunction &) + void Project(QuadratureFunction &qf) override; + virtual ~GradientGridFunctionCoefficient() { } }; @@ -1450,6 +1456,9 @@ public: /// Set the time for internally stored coefficients void SetTime(real_t t) override; + /// @copydoc Coefficient::Project(QuadratureFunction &) + void Project(QuadratureFunction &qf) override; + /// Reset the first term in the linear combination as a constant void SetAConst(real_t A) { a = NULL; aConst = A; } /// Return the first term in the linear combination @@ -1631,6 +1640,9 @@ public: /// Set the time for internally stored coefficients void SetTime(real_t t) override; + /// @copydoc Coefficient::Project(QuadratureFunction &) + void Project(QuadratureFunction &qf) override; + /// Reset the first term in the product as a constant void SetAConst(real_t A) { a = NULL; aConst = A; } /// Return the first term in the product @@ -1679,6 +1691,9 @@ public: /// Set the time for internally stored coefficients void SetTime(real_t t) override; + /// @copydoc Coefficient::Project(QuadratureFunction &) + void Project(QuadratureFunction &qf) override; + /// Reset the numerator in the ratio as a constant void SetAConst(real_t A) { a = NULL; aConst = A; } /// Return the numerator of the ratio @@ -1771,6 +1786,9 @@ public: /// Evaluate the coefficient at @a ip. real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override; + + /// @copydoc Coefficient::Project(QuadratureFunction &) + void Project(QuadratureFunction &qf) override; }; /// Scalar coefficient defined as a cross product of two vectors in the xy-plane. diff --git a/fem/complex_fem.cpp b/fem/complex_fem.cpp index 48a2437498..b6bfa4f1fa 100644 --- a/fem/complex_fem.cpp +++ b/fem/complex_fem.cpp @@ -11,14 +11,15 @@ #include "complex_fem.hpp" #include "../general/forall.hpp" +#include "../general/text.hpp" using namespace std; namespace mfem { -ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes) - : Vector(2*(fes->GetVSize())) +ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *f) + : Vector(2*(f->GetVSize())), fes(f), fec_owned(NULL) { UseDevice(true); this->Vector::operator=(0.0); @@ -28,12 +29,88 @@ ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes) gfi = new GridFunction(); gfi->MakeRef(fes, *this, fes->GetVSize()); + + fes_sequence = fes->GetSequence(); +} + +ComplexGridFunction::ComplexGridFunction(Mesh *m, std::istream &input) + : Vector(), fes(NULL), fec_owned(NULL) +{ + string buff; + + // Grid functions are stored on the device + UseDevice(true); + + input >> std::ws; + getline(input, buff); // 'ComplexGridFunction' + filter_dos(buff); + if (buff != "ComplexGridFunction") + { + MFEM_ABORT("unrecognized file header: " << buff); + } + + fes = new FiniteElementSpace; + fec_owned = fes->Load(m, input); + + skip_comment_lines(input, '#'); + istream::int_type next_char = input.peek(); + if (next_char == 'N') // First letter of "NURBS_patches" + { + getline(input, buff); + filter_dos(buff); + if (buff == "NURBS_patches") + { + MFEM_ABORT("NURBS not yet supported with ComplexGridFunction objects"); + } + else + { + MFEM_ABORT("unknown section: " << buff); + } + } + else + { + Vector::Load(input, 2*fes->GetVSize()); + + // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering + if (fes->Nonconforming() && + fes->GetMesh()->ncmesh->IsLegacyLoaded()) + { + // LegacyNCReorder(); + MFEM_ABORT("LegacyNCReorder not supported for " + "ComplexGridFunction objects"); + } + } + + gfr = new GridFunction(); + gfr->MakeRef(fes, *this, 0); + + gfi = new GridFunction(); + gfi->MakeRef(fes, *this, fes->GetVSize()); + + fes_sequence = fes->GetSequence(); +} + +void ComplexGridFunction::Destroy() +{ + delete gfr; delete gfi; + + if (fec_owned) + { + delete fes; + delete fec_owned; + fec_owned = NULL; + } } void ComplexGridFunction::Update() { - FiniteElementSpace *fes = gfr->FESpace(); + if (fes->GetSequence() == fes_sequence) + { + return; // space and grid function are in sync, no-op + } + fes_sequence = fes->GetSequence(); + const int vsize = fes->GetVSize(); const Operator *T = fes->GetUpdateOperator(); @@ -84,6 +161,17 @@ ComplexGridFunction::Update() } } +int ComplexGridFunction::VectorDim() const +{ + const FiniteElement *fe = fes->GetTypicalFE(); + if (!fe || fe->GetRangeType() == FiniteElement::SCALAR) + { + return fes->GetVDim(); + } + return fes->GetVDim()*std::max(fes->GetMesh()->SpaceDimension(), + fe->GetRangeDim()); +} + void ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff, Coefficient &imag_coeff) @@ -149,6 +237,35 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient gfi->SyncAliasMemory(*this); } +void ComplexGridFunction::Save(std::ostream &os) const +{ + os << "ComplexGridFunction\n"; + fes->Save(os); + os << '\n'; + if (fes->GetOrdering() == Ordering::byNODES) + { + Vector::Print(os, 1); + } + else + { + Vector::Print(os, fes->GetVDim()); + } + os.flush(); +} + +void ComplexGridFunction::Save(const char *fname, int precision) const +{ + ofstream ofs(fname); + ofs.precision(precision); + Save(ofs); +} + +std::ostream &operator<<(std::ostream &os, const ComplexGridFunction &sol) +{ + sol.Save(os); + return os; +} + ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes, ComplexOperator::Convention convention) @@ -654,8 +771,8 @@ SesquilinearForm::Update(FiniteElementSpace *nfes) #ifdef MFEM_USE_MPI -ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes) - : Vector(2*(pfes->GetVSize())) +ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pf) + : Vector(2*(pf->GetVSize())), pfes(pf), fec_owned(NULL) { UseDevice(true); this->Vector::operator=(0.0); @@ -665,12 +782,105 @@ ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes) pgfi = new ParGridFunction(); pgfi->MakeRef(pfes, *this, pfes->GetVSize()); + + fes_sequence = pfes->GetSequence(); +} + +ParComplexGridFunction::ParComplexGridFunction(ParMesh *m, std::istream &input) + : Vector(), pfes(NULL), fec_owned(NULL) +{ + string buff; + + // Grid functions are stored on the device + UseDevice(true); + + input >> std::ws; + getline(input, buff); // 'ParComplexGridFunction' + filter_dos(buff); + if (buff != "ParComplexGridFunction") + { + MFEM_ABORT("unrecognized file header: " << buff); + } + + FiniteElementSpace *fes = new FiniteElementSpace; + fec_owned = fes->Load(m, input); + + pfes = new ParFiniteElementSpace(m, fec_owned, fes->GetVDim(), + fes->GetOrdering()); + + delete fes; + + skip_comment_lines(input, '#'); + istream::int_type next_char = input.peek(); + if (next_char == 'N') // First letter of "NURBS_patches" + { + getline(input, buff); + filter_dos(buff); + if (buff == "NURBS_patches") + { + MFEM_ABORT("NURBS not yet supported with ComplexGridFunction objects"); + } + else + { + MFEM_ABORT("unknown section: " << buff); + } + } + else + { + int vsize = pfes->GetVSize(); + Vector::Load(input, 2*vsize); + + real_t *data_ = const_cast(HostRead()); + for (int i = 0; i < vsize; i++) + { + if (pfes->GetDofSign(i) < 0) + { + data_[i] = -data_[i]; + data_[i+vsize] = -data_[i+vsize]; + } + } + + + // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering + if (pfes->Nonconforming() && + pfes->GetMesh()->ncmesh->IsLegacyLoaded()) + { + // LegacyNCReorder(); + MFEM_ABORT("LegacyNCReorder not supported for " + "ComplexGridFunction objects"); + } + } + + pgfr = new ParGridFunction(); + pgfr->MakeRef(pfes, *this, 0); + + pgfi = new ParGridFunction(); + pgfi->MakeRef(pfes, *this, pfes->GetVSize()); + + fes_sequence = pfes->GetSequence(); +} + +void ParComplexGridFunction::Destroy() +{ + delete pgfr; delete pgfi; + + if (fec_owned) + { + delete pfes; + delete fec_owned; + fec_owned = NULL; + } } void ParComplexGridFunction::Update() { - ParFiniteElementSpace *pfes = pgfr->ParFESpace(); + if (pfes->GetSequence() == fes_sequence) + { + return; // space and grid function are in sync, no-op + } + fes_sequence = pfes->GetSequence(); + const int vsize = pfes->GetVSize(); const Operator *T = pfes->GetUpdateOperator(); @@ -719,6 +929,17 @@ ParComplexGridFunction::Update() } } +int ParComplexGridFunction::VectorDim() const +{ + const FiniteElement *fe = pfes->GetTypicalFE(); + if (!fe || fe->GetRangeType() == FiniteElement::SCALAR) + { + return pfes->GetVDim(); + } + return pfes->GetVDim()*std::max(pfes->GetMesh()->SpaceDimension(), + fe->GetRangeDim()); +} + void ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff, Coefficient &imag_coeff) @@ -789,7 +1010,6 @@ ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient void ParComplexGridFunction::Distribute(const Vector *tv) { - ParFiniteElementSpace *pfes = pgfr->ParFESpace(); const int tvsize = pfes->GetTrueVSize(); tv->Read(); @@ -807,7 +1027,6 @@ ParComplexGridFunction::Distribute(const Vector *tv) void ParComplexGridFunction::ParallelProject(Vector &tv) const { - ParFiniteElementSpace *pfes = pgfr->ParFESpace(); const int tvsize = pfes->GetTrueVSize(); tv.Write(); @@ -825,6 +1044,60 @@ ParComplexGridFunction::ParallelProject(Vector &tv) const tvi.SyncAliasMemory(tv); } +void ParComplexGridFunction::Save(std::ostream &os) const +{ + os << "ParComplexGridFunction\n"; + pfes->Save(os); + os << '\n'; + + int vsize = pfes->GetVSize(); + real_t *data_ = const_cast(HostRead()); + for (int i = 0; i < vsize; i++) + { + if (pfes->GetDofSign(i) < 0) + { + data_[i] = -data_[i]; + data_[i+vsize] = -data_[i+vsize]; + } + } + + if (pfes->GetOrdering() == Ordering::byNODES) + { + Vector::Print(os, 1); + } + else + { + Vector::Print(os, pfes->GetVDim()); + } + + for (int i = 0; i < vsize; i++) + { + if (pfes->GetDofSign(i) < 0) + { + data_[i] = -data_[i]; + data_[i+vsize] = -data_[i+vsize]; + } + } + + os.flush(); +} + +void ParComplexGridFunction::Save(const char *fname, int precision) const +{ + int rank = pfes->GetMyRank(); + ostringstream fname_with_suffix; + fname_with_suffix << fname << "." << setfill('0') << setw(6) << rank; + ofstream ofs(fname_with_suffix.str().c_str()); + ofs.precision(precision); + Save(ofs); +} + +std::ostream &operator<<(std::ostream &os, const ParComplexGridFunction &sol) +{ + sol.Save(os); + return os; +} + ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes, ComplexOperator::Convention diff --git a/fem/complex_fem.hpp b/fem/complex_fem.hpp index f1d363d2d0..9b66b6c177 100644 --- a/fem/complex_fem.hpp +++ b/fem/complex_fem.hpp @@ -35,15 +35,53 @@ private: GridFunction * gfi; protected: - void Destroy() { delete gfr; delete gfi; } + /// FE space on which the grid function lives. Owned if #fec_owned + /// is not NULL. + FiniteElementSpace *fes; + + /** @brief Used when the grid function is read from a file. It can also be + set explicitly, see MakeOwner(). + + If not NULL, this pointer is owned by the ComplexGridFunction. */ + FiniteElementCollection *fec_owned; + + long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence + + void Destroy(); public: /** @brief Construct a ComplexGridFunction associated with the FiniteElementSpace @a *f. */ ComplexGridFunction(FiniteElementSpace *f); + /** @brief Construct a ComplexGridFunction on the given Mesh, using the data + from @a input. + + The content of @a input should be in the format created by the method + Save(). The reconstructed FiniteElementSpace and FiniteElementCollection + are owned by the ComplexGridFunction. */ + ComplexGridFunction(Mesh *m, std::istream &input); + void Update(); + /** Return update counter, similar to Mesh::GetSequence(). Used to + check if it is up to date with the space. */ + long GetSequence() const { return fes_sequence; } + + /// Make the ComplexGridFunction the owner of #fec_owned and #fes. + /** If the new FiniteElementCollection, @a fec_, is NULL, ownership + of #fec_owned and #fes is taken away. */ + void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; } + + /// Returns a pointer to the FiniteElementCollection used to + /// construct this ComplexGridFunction if this class owns that + /// object. Otherwise this function will return NULL. + FiniteElementCollection *OwnFEC() { return fec_owned; } + + /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the + /// underlying #fes + int VectorDim() const; + /// Assign constant values to the ComplexGridFunction data. ComplexGridFunction &operator=(const std::complex & value) { *gfr = value.real(); *gfi = value.imag(); return *this; } @@ -63,8 +101,8 @@ public: VectorCoefficient &imag_coeff, Array &attr); - FiniteElementSpace *FESpace() { return gfr->FESpace(); } - const FiniteElementSpace *FESpace() const { return gfr->FESpace(); } + FiniteElementSpace *FESpace() { return fes; } + const FiniteElementSpace *FESpace() const { return fes; } GridFunction & real() { return *gfr; } GridFunction & imag() { return *gfi; } @@ -79,11 +117,52 @@ public: /// @a gfr and @a gfi to match the ComplexGridFunction. void SyncAlias() { gfr->SyncAliasMemory(*this); gfi->SyncAliasMemory(*this); } + /// @brief Returns ||u_ex - u_h||_L2 for complex-valued scalar fields + /// + /// @see GridFunction::ComputeL2Error(Coefficient &exsol, + /// const IntegrationRule *irs[], + /// const Array *elems) const + /// for more detailed documentation. + virtual real_t ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli, + const IntegrationRule *irs[] = NULL) const + { + real_t err_r = gfr->ComputeL2Error(exsolr, irs); + real_t err_i = gfi->ComputeL2Error(exsoli, irs); + return sqrt(err_r * err_r + err_i * err_i); + } + + /// @brief Returns ||u_ex - u_h||_L2 for complex-valued vector fields + /// + /// @see GridFunction::ComputeL2Error(VectorCoefficient &exsol, + /// const IntegrationRule *irs[], + /// const Array *elems) const + /// for more detailed documentation. + virtual real_t ComputeL2Error(VectorCoefficient &exsolr, + VectorCoefficient &exsoli, + const IntegrationRule *irs[] = NULL, + Array *elems = NULL) const + { + real_t err_r = gfr->ComputeL2Error(exsolr, irs, elems); + real_t err_i = gfi->ComputeL2Error(exsoli, irs, elems); + return sqrt(err_r * err_r + err_i * err_i); + } + + /// Save the ComplexGridFunction to an output stream. + virtual void Save(std::ostream &out) const; + + /// Save the ComplexGridFunction to a file + /** The given @a precision will be used for ASCII output. */ + virtual void Save(const char *fname, int precision=16) const; + /// Destroys the grid function. virtual ~ComplexGridFunction() { Destroy(); } }; +/** Overload operator<< for std::ostream and ComplexGridFunction; not valid + for the class ParComplexGridFunction */ +std::ostream &operator<<(std::ostream &out, const ComplexGridFunction &sol); + /** Class for a complex-valued linear form The @a convention argument in the class's constructor is documented in the @@ -345,12 +424,23 @@ public: class ParComplexGridFunction : public Vector { private: - ParGridFunction * pgfr; ParGridFunction * pgfi; protected: - void Destroy() { delete pgfr; delete pgfi; } + /// FE space on which the grid function lives. Owned if #fec_owned + /// is not NULL. + ParFiniteElementSpace *pfes; + + /** @brief Used when the grid function is read from a file. It can also be + set explicitly, see MakeOwner(). + + If not NULL, this pointer is owned by the ParComplexGridFunction. */ + FiniteElementCollection *fec_owned; + + long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence + + void Destroy(); public: @@ -358,8 +448,33 @@ public: ParFiniteElementSpace @a *pf. */ ParComplexGridFunction(ParFiniteElementSpace *pf); + /** @brief Construct a ParComplexGridFunction on a given ParMesh, + @a pmesh, reading from an std::istream. + + In the process, a ParFiniteElementSpace and a FiniteElementCollection are + constructed. The new ParComplexGridFunction assumes ownership of both. */ + ParComplexGridFunction(ParMesh *pmesh, std::istream &input); + void Update(); + /** Return update counter, similar to Mesh::GetSequence(). Used to + check if it is up to date with the space. */ + long GetSequence() const { return fes_sequence; } + + /// Make the ParComplexGridFunction the owner of #fec_owned and #pfes. + /** If the new FiniteElementCollection, @a fec_, is NULL, ownership + of #fec_owned and #pfes is taken away. */ + void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; } + + /// Returns a pointer to the FiniteElementCollection used to + /// construct this ParComplexGridFunction if this class owns that + /// object. Otherwise this function will return NULL. + FiniteElementCollection *OwnFEC() { return fec_owned; } + + /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the + /// underlying #pfes + int VectorDim() const; + /// Assign constant values to the ParComplexGridFunction data. ParComplexGridFunction &operator=(const std::complex & value) { *pgfr = value.real(); *pgfi = value.imag(); return *this; } @@ -385,11 +500,11 @@ public: /// Returns the vector restricted to the true dofs. void ParallelProject(Vector &tv) const; - FiniteElementSpace *FESpace() { return pgfr->FESpace(); } - const FiniteElementSpace *FESpace() const { return pgfr->FESpace(); } + FiniteElementSpace *FESpace() { return pfes; } + const FiniteElementSpace *FESpace() const { return pfes; } - ParFiniteElementSpace *ParFESpace() { return pgfr->ParFESpace(); } - const ParFiniteElementSpace *ParFESpace() const { return pgfr->ParFESpace(); } + ParFiniteElementSpace *ParFESpace() { return pfes; } + const ParFiniteElementSpace *ParFESpace() const { return pfes; } ParGridFunction & real() { return *pgfr; } ParGridFunction & imag() { return *pgfi; } @@ -402,17 +517,32 @@ public: /// Update the alias memory location of the real and imaginary /// ParGridFunction @a pgfr and @a pgfi to match the ParComplexGridFunction. - void SyncAlias() { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); } - + void SyncAlias() + { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); } + /// @brief Returns ||u_ex - u_h||_L2 in parallel for complex-valued + /// scalar fields + /// + /// @see GridFunction::ComputeL2Error(Coefficient &exsol, + /// const IntegrationRule *irs[], + /// const Array *elems) const + /// for more detailed documentation. virtual real_t ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli, - const IntegrationRule *irs[] = NULL) const + const IntegrationRule *irs[] = NULL, + Array *elems = NULL) const { - real_t err_r = pgfr->ComputeL2Error(exsolr, irs); - real_t err_i = pgfi->ComputeL2Error(exsoli, irs); - return sqrt(err_r * err_r + err_i * err_i); + real_t err_r = pgfr->ComputeL2Error(exsolr, irs, elems); + real_t err_i = pgfi->ComputeL2Error(exsoli, irs, elems); + return hypot(err_r, err_i); } + /// @brief Returns ||u_ex - u_h||_L2 in parallel for complex-valued + /// vector fields + /// + /// @see GridFunction::ComputeL2Error(VectorCoefficient &exsol, + /// const IntegrationRule *irs[], + /// const Array *elems) const + /// for more detailed documentation. virtual real_t ComputeL2Error(VectorCoefficient &exsolr, VectorCoefficient &exsoli, const IntegrationRule *irs[] = NULL, @@ -420,15 +550,28 @@ public: { real_t err_r = pgfr->ComputeL2Error(exsolr, irs, elems); real_t err_i = pgfi->ComputeL2Error(exsoli, irs, elems); - return sqrt(err_r * err_r + err_i * err_i); + return hypot(err_r, err_i); } + /// Save the local portion of the ParComplexGridFunction + /** This differs from the serial ComplexGridFunction::Save in that it + takes into account the signs of the local dofs. */ + void Save(std::ostream &out) const; + + /// Save the ParComplexGridFunction to files + /** Saves one file for each MPI rank. The files will be given suffixes + according to the MPI rank. The given @a precision will be used for ASCII + output. */ + void Save(const char *fname, int precision=16) const; /// Destroys grid function. virtual ~ParComplexGridFunction() { Destroy(); } }; +/** Overload operator<< for std::ostream and ParComplexGridFunction */ +std::ostream &operator<<(std::ostream &out, const ParComplexGridFunction &sol); + /** Class for a complex-valued, parallel linear form The @a convention argument in the class's constructor is documented in the diff --git a/fem/datacollection.cpp b/fem/datacollection.cpp index 10e8b77b55..e37f6ce184 100644 --- a/fem/datacollection.cpp +++ b/fem/datacollection.cpp @@ -310,9 +310,9 @@ void DataCollection::SaveField(const std::string &field_name) } } -void DataCollection::SaveQField(const std::string &q_field_name) +void DataCollection::SaveQField(const std::string &field_name) { - QFieldMapIterator it = q_field_map.find(q_field_name); + QFieldMapIterator it = q_field_map.find(field_name); if (it != q_field_map.end()) { SaveOneQField(it); @@ -780,6 +780,11 @@ void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_) high_order_output = high_order_output_; } +void ParaViewDataCollectionBase::SetBoundaryOutput(bool bdr_output_) +{ + bdr_output = bdr_output_; +} + void ParaViewDataCollectionBase::SetCompressionLevel(int compression_level_) { MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9, @@ -935,16 +940,19 @@ void ParaViewDataCollection::Save() std::string vtu_prefix = col_path + "/" + GenerateVTUPath() + "/"; // Save the local part of the mesh and grid functions fields to the local - // VTU file + // VTU file. Also save coefficient fields. { std::ofstream os(vtu_prefix + GenerateVTUFileName("proc", myid)); os.precision(precision); SaveDataVTU(os, levels_of_detail); } - // Save the local part of the quadrature function fields + // Save the local part of the quadrature function fields. for (const auto &qfield : q_field_map) { + MFEM_VERIFY(!bdr_output, + "QuadratureFunction output is not supported for " + "ParaViewDataCollection on domain boundary!"); const std::string &field_name = qfield.first; std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid)); qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel(), field_name); @@ -960,7 +968,7 @@ void ParaViewDataCollection::Save() std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data")); WritePVTUHeader(pvtu_out); - // Grid function fields + // Grid function fields and coefficient fields pvtu_out << "\n"; for (auto &field_it : field_map) { @@ -971,7 +979,24 @@ void ParaViewDataCollection::Save() << VTKComponentLabels(vec_dim) << " " << "format=\"" << GetDataFormatString() << "\" />\n"; } + for (auto &field_it : coeff_field_map) + { + int vec_dim = 1; + pvtu_out << "\n"; + } + for (auto &field_it : vcoeff_field_map) + { + int vec_dim = field_it.second->GetVDim(); + pvtu_out << "\n"; + } pvtu_out << "\n"; + // Element attributes pvtu_out << "\n"; pvtu_out << "\t\n"; os << "\n"; - mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel()); + mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel(), + bdr_output); // dump out the grid functions as point data os << "\n"; @@ -1077,8 +1103,21 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &os, int ref) // iterate over all grid functions for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it) { + MFEM_VERIFY(!bdr_output, + "GridFunction output is not supported for " + "ParaViewDataCollection on domain boundary!"); SaveGFieldVTU(os,ref,it); } + // save the coefficient functions + // iterate over all Coefficient and VectorCoefficient functions + for (const auto &kv : coeff_field_map) + { + SaveCoeffFieldVTU(os, ref, kv.first, *kv.second); + } + for (const auto &kv : vcoeff_field_map) + { + SaveVCoeffFieldVTU(os, ref, kv.first, *kv.second); + } os << "\n"; // close the mesh os << "\n"; // close the piece open in the PrintVTU method @@ -1101,7 +1140,6 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_, << "format=\"" << GetDataFormatString() << "\" >" << '\n'; if (vec_dim == 1) { - // scalar data for (int i = 0; i < mesh->GetNE(); i++) { RefG = GlobGeometryRefiner.Refine( @@ -1131,11 +1169,131 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_, } } } - - if (IsBinaryFormat()) + if (pv_data_format != VTKFormat::ASCII) { - WriteVTKEncodedCompressed(os,buf.data(),buf.size(),GetCompressionLevel()); - os << '\n'; + WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel()); + } + os << "" << std::endl; +} + +void ParaViewDataCollection::SaveCoeffFieldVTU(std::ostream &os, int ref_, + const std::string &name, Coefficient &coeff) +{ + RefinedGeometry *RefG; + real_t val; + std::vector buf; + int vec_dim = 1; + os << "" << '\n'; + { + // scalar data + if (!bdr_output) + { + for (int i = 0; i < mesh->GetNE(); i++) + { + RefG = GlobGeometryRefiner.Refine( + mesh->GetElementBaseGeometry(i), ref_, 1); + + ElementTransformation *eltrans = mesh->GetElementTransformation(i); + const IntegrationRule *ir = &RefG->RefPts; + for (int j = 0; j < ir->GetNPoints(); j++) + { + const IntegrationPoint &ip = ir->IntPoint(j); + eltrans->SetIntPoint(&ip); + val = coeff.Eval(*eltrans, ip); + WriteBinaryOrASCII(os, buf, val, "\n", pv_data_format); + } + } + } + else + { + for (int i = 0; i < mesh->GetNBE(); i++) + { + RefG = GlobGeometryRefiner.Refine( + mesh->GetBdrElementBaseGeometry(i), ref_, 1); + + ElementTransformation *eltrans = mesh->GetBdrElementTransformation(i); + const IntegrationRule *ir = &RefG->RefPts; + for (int j = 0; j < ir->GetNPoints(); j++) + { + const IntegrationPoint &ip = ir->IntPoint(j); + eltrans->SetIntPoint(&ip); + val = coeff.Eval(*eltrans, ip); + WriteBinaryOrASCII(os, buf, val, "\n", pv_data_format); + } + } + } + } + if (pv_data_format != VTKFormat::ASCII) + { + WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel()); + } + os << "" << std::endl; +} + +void ParaViewDataCollection::SaveVCoeffFieldVTU(std::ostream &os, int ref_, + const std::string &name, VectorCoefficient &coeff) +{ + RefinedGeometry *RefG; + Vector val; + std::vector buf; + int vec_dim = coeff.GetVDim(); + os << "" << '\n'; + { + // vector data + if (!bdr_output) + { + for (int i = 0; i < mesh->GetNE(); i++) + { + RefG = GlobGeometryRefiner.Refine( + mesh->GetElementBaseGeometry(i), ref_, 1); + + ElementTransformation *eltrans = mesh->GetElementTransformation(i); + const IntegrationRule *ir = &RefG->RefPts; + for (int j = 0; j < ir->GetNPoints(); j++) + { + const IntegrationPoint &ip = ir->IntPoint(j); + eltrans->SetIntPoint(&ip); + coeff.Eval(val, *eltrans, ip); + for (int jj = 0; jj < val.Size(); jj++) + { + WriteBinaryOrASCII(os, buf, val(jj), " ", pv_data_format); + } + if (pv_data_format == VTKFormat::ASCII) { os << '\n'; } + } + } + } + else + { + for (int i = 0; i < mesh->GetNBE(); i++) + { + RefG = GlobGeometryRefiner.Refine( + mesh->GetBdrElementBaseGeometry(i), ref_, 1); + + ElementTransformation *eltrans = mesh->GetBdrElementTransformation(i); + const IntegrationRule *ir = &RefG->RefPts; + for (int j = 0; j < ir->GetNPoints(); j++) + { + const IntegrationPoint &ip = ir->IntPoint(j); + eltrans->SetIntPoint(&ip); + coeff.Eval(val, *eltrans, ip); + for (int jj = 0; jj < val.Size(); jj++) + { + WriteBinaryOrASCII(os, buf, val(jj), " ", pv_data_format); + } + if (pv_data_format == VTKFormat::ASCII) { os << '\n'; } + } + } + } + } + if (pv_data_format != VTKFormat::ASCII) + { + WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel()); } os << "" << std::endl; } diff --git a/fem/datacollection.hpp b/fem/datacollection.hpp index a4a376f8b1..e82830304d 100644 --- a/fem/datacollection.hpp +++ b/fem/datacollection.hpp @@ -133,6 +133,7 @@ private: /// A collection of named QuadratureFunctions typedef NamedFieldsMap QFieldMap; + public: typedef GFieldMap::MapType FieldMapType; typedef GFieldMap::iterator FieldMapIterator; @@ -249,10 +250,9 @@ public: { field_map.Deregister(field_name, own_data); } /// Add a QuadratureFunction to the collection. - virtual void RegisterQField(const std::string& q_field_name, + virtual void RegisterQField(const std::string& field_name, QuadratureFunction *qf) - { q_field_map.Register(q_field_name, qf, own_data); } - + { q_field_map.Register(field_name, qf, own_data); } /// Remove a QuadratureFunction from the collection virtual void DeregisterQField(const std::string& field_name) @@ -280,13 +280,13 @@ public: #endif /// Check if a QuadratureFunction with the given name is in the collection. - bool HasQField(const std::string& q_field_name) const - { return q_field_map.Has(q_field_name); } + bool HasQField(const std::string& field_name) const + { return q_field_map.Has(field_name); } /// Get a pointer to a QuadratureFunction in the collection. /** Returns NULL if @a field_name is not in the collection. */ - QuadratureFunction *GetQField(const std::string& q_field_name) - { return q_field_map.Get(q_field_name); } + QuadratureFunction *GetQField(const std::string& field_name) + { return q_field_map.Get(field_name); } /// Get a const reference to the internal field map. /** The keys in the map are the field names and the values are pointers to @@ -302,11 +302,13 @@ public: /// Get a pointer to the mesh in the collection Mesh *GetMesh() { return mesh; } + /// Set/change the mesh associated with the collection /** When passed a Mesh, assumes the serial case: MPI rank id is set to 0 and MPI num_procs is set to 1. When passed a ParMesh, MPI info from the ParMesh is used to set the DataCollection's MPI rank and num_procs. */ virtual void SetMesh(Mesh *new_mesh); + #ifdef MFEM_USE_MPI /// Set/change the mesh associated with the collection. /** For this case, @a comm is used to set the DataCollection's MPI rank id @@ -369,8 +371,7 @@ public: /// Save one field, assuming the collection directory already exists. virtual void SaveField(const std::string &field_name); /// Save one q-field, assuming the collection directory already exists. - virtual void SaveQField(const std::string &q_field_name); - + virtual void SaveQField(const std::string &field_name); /// Load the collection. Not implemented in the base class DataCollection. virtual void Load(int cycle_ = 0); @@ -510,7 +511,9 @@ protected: int compression_level = -1; bool high_order_output = false; bool restart_mode = false; + bool bdr_output = false; VTKFormat pv_data_format = VTKFormat::BINARY; + public: ParaViewDataCollectionBase(const std::string &name, Mesh *mesh); @@ -543,6 +546,10 @@ public: /// Reading high-order data requires ParaView 5.5 or later. void SetHighOrderOutput(bool high_order_output_); + /// @brief Configures collection to save only fields evaluated on boundaries of + /// the mesh. + void SetBoundaryOutput(bool bdr_output_); + /// If compression is enabled, return the compression level, else return 0. int GetCompressionLevel() const; @@ -564,8 +571,6 @@ public: /// /// If restart is enabled, new writes will preserve timestep metadata for any /// solutions prior to the currently defined time. - /// - /// Initially, restart mode is disabled. void UseRestartMode(bool restart_mode_); }; @@ -575,11 +580,23 @@ class ParaViewDataCollection : public ParaViewDataCollectionBase private: std::fstream pvd_stream; + /// A collection of named Coefficients and VectorCoefficients + using CoeffFieldMap = NamedFieldsMap; + using VCoeffFieldMap = NamedFieldsMap; + + /** A FieldMap mapping registered names to Coefficient and VectorCoefficient + pointers. */ + CoeffFieldMap coeff_field_map; + VCoeffFieldMap vcoeff_field_map; protected: void WritePVTUHeader(std::ostream &out); void WritePVTUFooter(std::ostream &out, const std::string &vtu_prefix); void SaveDataVTU(std::ostream &out, int ref); void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it); + void SaveCoeffFieldVTU(std::ostream& out, int ref_, const std::string &name, + Coefficient &coeff); + void SaveVCoeffFieldVTU(std::ostream& out, int ref_, const std::string &name, + VectorCoefficient& coeff); const char *GetDataFormatString() const; const char *GetDataTypeString() const; @@ -598,6 +615,25 @@ public: ParaViewDataCollection(const std::string& collection_name, Mesh *mesh_ = nullptr); + /// Get a const reference to the internal coefficient-field map. + const typename CoeffFieldMap::MapType &GetCoeffFieldMap() const + { return coeff_field_map.GetMap(); } + const typename VCoeffFieldMap::MapType &GetVCoeffFieldMap() const + { return vcoeff_field_map.GetMap(); } + + /// Add a Coefficient or VectorCoefficient to the collection. + void RegisterCoeffField(const std::string& field_name, Coefficient *coeff) + { coeff_field_map.Register(field_name, coeff, own_data); } + void RegisterVCoeffField(const std::string& field_name, + VectorCoefficient *vcoeff) + { vcoeff_field_map.Register(field_name, vcoeff, own_data); } + + /// Remove a Coefficient or VectorCoefficient from the collection + void DeregisterCoeffField(const std::string& field_name) + { coeff_field_map.Deregister(field_name, own_data); } + void DeregisterVCoeffField(const std::string& field_name) + { vcoeff_field_map.Deregister(field_name, own_data); } + /// Save the collection - the directory name is constructed based on the /// cycle value void Save() override; diff --git a/fem/dfem/doperator.hpp b/fem/dfem/doperator.hpp index 8f4f8a9807..35d3efdf60 100644 --- a/fem/dfem/doperator.hpp +++ b/fem/dfem/doperator.hpp @@ -211,8 +211,8 @@ private: /// /// The operator is constructed with solution fields that it will act on and /// parameter fields that define coefficients. Quadrature functions are added by -/// e.g. using AddDomainIntegrator() which specify how the operator evaluates f -/// those functionas and parameters at quadrature points. +/// e.g. using AddDomainIntegrator() which specify how the operator evaluates +/// those functions and parameters at quadrature points. /// /// Derivatives can be computed by obtaining a DerivativeOperator using /// GetDerivative(). @@ -231,24 +231,71 @@ public: const std::vector ¶meters, const ParMesh &mesh); + /// MultLevel enum to indicate if the T->L Operators are used in the + /// Mult method. + enum MultLevel + { + TVECTOR, + LVECTOR + }; + + /// @brief Set the MultLevel mode for the DifferentiableOperator. + /// The default is TVECTOR, which means that the Operator will use + /// T->L before Mult and L->T Operators after. + void SetMultLevel(MultLevel level) + { + mult_level = level; + } + /// @brief Compute the action of the operator on a given vector. /// - /// @param solutions_t The solution vector in which to compute the action. - /// This has to be a T-dof vector. - /// @param result_t Result vector of the action of the operator on - /// solutions_t. The result is a T-dof vector. - void Mult(const Vector &solutions_t, Vector &result_t) const override + /// @param solutions_in The solution vector in which to compute the action. + /// This has to be a T-dof vector if MultLevel is set to TVECTOR, or L-dof + /// Vector if MultLevel is set to LVECTOR. + /// @param result_in Result vector of the action of the operator on + /// solutions. The result is a T-dof vector or L-dof vector depending on + /// the MultLevel. + void Mult(const Vector &solutions_in, Vector &result_in) const override { MFEM_ASSERT(!action_callbacks.empty(), "no integrators have been set"); - prolongation(solutions, solutions_t, solutions_l); - residual_l = 0.0; - for (auto &action : action_callbacks) + + if (mult_level == MultLevel::LVECTOR) { - action(solutions_l, parameters_l, residual_l); + get_lvectors(solutions, solutions_in, solutions_l); + result_in = 0.0; + for (auto &action : action_callbacks) + { + action(solutions_l, parameters_l, result_in); + } + } + else + { + prolongation(solutions, solutions_in, solutions_l); + residual_l = 0.0; + for (auto &action : action_callbacks) + { + action(solutions_l, parameters_l, residual_l); + } + prolongation_transpose(residual_l, result_in); } - prolongation_transpose(residual_l, result_t); } + /// @brief Add an integrator to the operator. + /// Called only from AddDomainIntegrator() and AddBoundaryIntegrator(). + template < + typename entity_t, + typename qfunc_t, + typename input_t, + typename output_t, + typename derivative_ids_t> + void AddIntegrator( + qfunc_t &qfunc, + input_t inputs, + output_t outputs, + const IntegrationRule &integration_rule, + const Array &attributes, + derivative_ids_t derivative_ids); + /// @brief Add a domain integrator to the operator. /// /// @param qfunc The quadrature function to be added. @@ -274,6 +321,31 @@ public: const Array &domain_attributes, derivative_ids_t derivative_ids = std::make_index_sequence<0> {}); + /// @brief Add a boundary integrator to the operator. + /// + /// @param qfunc The quadrature function to be added. + /// @param inputs Tuple of FieldOperators for the inputs of the quadrature + /// function. + /// @param outputs Tuple of FieldOperators for the outputs of the quadrature + /// function. + /// @param integration_rule IntegrationRule to use with this integrator. + /// @param boundary_attributes Boundary attributes marker array indicating over + /// which attributes this integrator will integrate over. + /// @param derivative_ids Derivatives to be made available for this + /// integrator. + template < + typename qfunc_t, + typename input_t, + typename output_t, + typename derivative_ids_t = decltype(std::make_index_sequence<0> {})> + void AddBoundaryIntegrator( + qfunc_t &qfunc, + input_t inputs, + output_t outputs, + const IntegrationRule &integration_rule, + const Array &boundary_attributes, + derivative_ids_t derivative_ids = std::make_index_sequence<0> {}); + /// @brief Set the parameters for the operator. /// /// This has to be called before using Mult() or MultTranspose(). @@ -345,6 +417,8 @@ public: private: const ParMesh &mesh; + MultLevel mult_level = TVECTOR; + std::vector action_callbacks; std::map> derivative_action_callbacks; @@ -354,7 +428,6 @@ private: std::vector> assemble_derivative_hypreparmatrix_callbacks; - std::vector solutions; std::vector parameters; // solutions and parameters @@ -391,7 +464,52 @@ void DifferentiableOperator::AddDomainIntegrator( const Array &domain_attributes, derivative_ids_t derivative_ids) { - using entity_t = Entity::Element; + AddIntegrator( + qfunc, inputs, outputs, integration_rule, domain_attributes, derivative_ids); +} + +template < + typename qfunc_t, + typename input_t, + typename output_t, + typename derivative_ids_t> +void DifferentiableOperator::AddBoundaryIntegrator( + qfunc_t &qfunc, + input_t inputs, + output_t outputs, + const IntegrationRule &integration_rule, + const Array &boundary_attributes, + derivative_ids_t derivative_ids) +{ + + if (mesh.GetNFbyType(FaceType::Boundary) != mesh.GetNBE()) + { + MFEM_ABORT("AddBoundaryIntegrator on meshes with interior boundaries is not supported."); + } + AddIntegrator( + qfunc, inputs, outputs, integration_rule, boundary_attributes, derivative_ids); +} + +template < + typename entity_t, + typename qfunc_t, + typename input_t, + typename output_t, + typename derivative_ids_t> +void DifferentiableOperator::AddIntegrator( + qfunc_t &qfunc, + input_t inputs, + output_t outputs, + const IntegrationRule &integration_rule, + const Array &attributes, + derivative_ids_t derivative_ids) +{ + if constexpr (!(std::is_same_v || + std::is_same_v)) + { + static_assert(dfem::always_false, + "entity type not supported in AddIntegrator"); + } static constexpr size_t num_inputs = tuple_size::value; @@ -445,32 +563,44 @@ void DifferentiableOperator::AddDomainIntegrator( auto output_to_field = create_descriptors_to_fields_map(fields, outputs); - // TODO: factor out - std::vector inputs_vdim(num_inputs); - for_constexpr([&](auto i) + const Array *elem_attributes = nullptr; + if constexpr (std::is_same_v) { - inputs_vdim[i] = get(inputs).vdim; - }); - - - Array elem_attributes; - elem_attributes.SetSize(mesh.GetNE()); - for (int i = 0; i < mesh.GetNE(); ++i) + elem_attributes = &mesh.GetElementAttributes(); + } + else if constexpr (std::is_same_v) { - elem_attributes[i] = mesh.GetAttribute(i); + elem_attributes = &mesh.GetBdrFaceAttributes(); } const auto output_fop = get<0>(outputs); test_space_field_idx = FindIdx(output_fop.GetFieldId(), fields); bool use_sum_factorization = false; - auto entity_element_type = - Element::TypeFromGeometry(mesh.GetTypicalElementGeometry()); - if ((entity_element_type == Element::QUADRILATERAL || - entity_element_type == Element::HEXAHEDRON) && - use_tensor_product_structure == true) + Element::Type entity_element_type; + if constexpr (std::is_same_v) { - use_sum_factorization = true; + entity_element_type = + Element::TypeFromGeometry(mesh.GetTypicalElementGeometry()); + + if ((entity_element_type == Element::QUADRILATERAL || + entity_element_type == Element::HEXAHEDRON) && + use_tensor_product_structure == true) + { + use_sum_factorization = true; + } + } + else if constexpr (std::is_same_v) + { + entity_element_type = + Element::TypeFromGeometry(mesh.GetTypicalFaceGeometry()); + + if ((entity_element_type == Element::SEGMENT || + entity_element_type == Element::QUADRILATERAL) && + use_tensor_product_structure == true) + { + use_sum_factorization = true; + } } ElementDofOrdering element_dof_ordering = ElementDofOrdering::NATIVE; @@ -508,8 +638,17 @@ void DifferentiableOperator::AddDomainIntegrator( prolongation_transpose = get_prolongation_transpose( fields[test_space_field_idx], output_fop, mesh.GetComm()); - const int dimension = mesh.Dimension(); - [[maybe_unused]] const int num_elements = GetNumEntities(mesh); + int dimension; + if constexpr (std::is_same_v) + { + dimension = mesh.Dimension(); + } + else if constexpr (std::is_same_v) + { + dimension = mesh.Dimension() - 1; + } + + [[maybe_unused]] const int num_elements = GetNumEntities(mesh); const int num_entities = GetNumEntities(mesh); const int num_qp = integration_rule.GetNPoints(); @@ -583,6 +722,12 @@ void DifferentiableOperator::AddDomainIntegrator( thread_blocks.z = 1; } } + else if (dimension == 1) + { + thread_blocks.x = q1d; + thread_blocks.y = 1; + thread_blocks.z = 1; + } action_callbacks.push_back( // Explicitly capture everything we need, so we can make explicit choice @@ -598,7 +743,7 @@ void DifferentiableOperator::AddDomainIntegrator( test_vdim, // int (= output_fop.vdim) test_op_dim, // int (derived from output_fop) inputs, // mfem::future::tuple - domain_attributes, // Array + attributes, // Array ir_weights, // DeviceTensor use_sum_factorization, // bool input_dtq_maps, // std::array @@ -631,13 +776,13 @@ void DifferentiableOperator::AddDomainIntegrator( action_shmem_info.field_sizes, num_entities); - const bool has_attr = domain_attributes.Size() > 0; - const auto d_domain_attr = domain_attributes.Read(); - const auto d_elem_attr = elem_attributes.Read(); + const bool has_attr = attributes.Size() > 0; + const auto d_attr = attributes.Read(); + const auto d_elem_attr = elem_attributes->Read(); forall([=] MFEM_HOST_DEVICE (int e, void *shmem) { - if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; } + if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; } auto [input_dtq_shmem, output_dtq_shmem, fields_shmem, input_shmem, residual_shmem, scratch_shmem] = @@ -707,7 +852,7 @@ void DifferentiableOperator::AddDomainIntegrator( test_vdim, // int (= output_fop.vdim) test_op_dim, // int (derived from output_fop) inputs, // mfem::future::tuple - domain_attributes, // Array + attributes, // Array ir_weights, // DeviceTensor use_sum_factorization, // bool input_dtq_maps, // std::array @@ -745,14 +890,14 @@ void DifferentiableOperator::AddDomainIntegrator( shmem_info.direction_size, num_entities); - const auto d_elem_attr = elem_attributes.Read(); - const bool has_attr = domain_attributes.Size() > 0; - const auto d_domain_attr = domain_attributes.Read(); + const bool has_attr = attributes.Size() > 0; + const auto d_attr = attributes.Read(); + const auto d_elem_attr = elem_attributes->Read(); derivative_action_e = 0.0; forall([=] MFEM_HOST_DEVICE (int e, real_t *shmem) { - if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; } + if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; } auto [input_dtq_shmem, output_dtq_shmem, fields_shmem, direction_shmem, input_shmem, diff --git a/fem/dfem/integrate.hpp b/fem/dfem/integrate.hpp index 8c17cb0c3b..3731d922b5 100644 --- a/fem/dfem/integrate.hpp +++ b/fem/dfem/integrate.hpp @@ -95,6 +95,85 @@ void map_quadrature_data_to_fields_impl( } } +template +MFEM_HOST_DEVICE +void map_quadrature_data_to_fields_tensor_impl_1d( + DeviceTensor<2, real_t> &y, + const DeviceTensor<3, real_t> &f, + const output_t &output, + const DofToQuadMap &dtq, + std::array, 6> &scratch_mem) +{ + [[maybe_unused]] auto B = dtq.B; + [[maybe_unused]] auto G = dtq.G; + + if constexpr (is_value_fop>::value) + { + const auto [q1d, unused, d1d] = B.GetShape(); + const int vdim = output.vdim; + const int test_dim = output.size_on_qp / vdim; + + auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d); + auto yd = Reshape(&y(0, 0), d1d, vdim); + + for (int vd = 0; vd < vdim; vd++) + { + MFEM_FOREACH_THREAD(dx, x, d1d) + { + real_t acc = 0.0; + for (int qx = 0; qx < q1d; qx++) + { + acc += fqp(vd, 0, qx) * B(qx, 0, dx); + } + yd(dx, vd) = acc; + } + } + MFEM_SYNC_THREAD; + } + else if constexpr (is_gradient_fop>::value) + { + const auto [q1d, unused, d1d] = G.GetShape(); + const int vdim = output.vdim; + const int test_dim = output.size_on_qp / vdim; + auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d); + auto yd = Reshape(&y(0, 0), d1d, vdim); + + for (int vd = 0; vd < vdim; vd++) + { + MFEM_FOREACH_THREAD(dx, x, d1d) + { + real_t acc = 0.0; + for (int qx = 0; qx < q1d; qx++) + { + acc += fqp(vd, 0, qx) * G(qx, 0, dx); + } + yd(dx, vd) = acc; + } + } + MFEM_SYNC_THREAD; + } + else if constexpr (is_identity_fop>::value) + { + const auto [q1d, unused, d1d] = B.GetShape(); + auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d); + auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d); + + for (int sq = 0; sq < output.size_on_qp; sq++) + { + MFEM_FOREACH_THREAD(qx, x, q1d) + { + yqp(sq, qx) = fqp(sq, qx); + } + MFEM_SYNC_THREAD; + } + } + else + { + MFEM_ABORT("quadrature data mapping to field is not implemented for" + " this field descriptor with sum factorization on tensor product elements"); + } +} + template MFEM_HOST_DEVICE void map_quadrature_data_to_fields_tensor_impl_2d( @@ -431,7 +510,11 @@ void map_quadrature_data_to_fields( { if (use_sum_factorization) { - if (dimension == 2) + if (dimension == 1) + { + map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem); + } + else if (dimension == 2) { map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem); } diff --git a/fem/dfem/interpolate.hpp b/fem/dfem/interpolate.hpp index 09c4ddc763..415df988bb 100644 --- a/fem/dfem/interpolate.hpp +++ b/fem/dfem/interpolate.hpp @@ -338,6 +338,92 @@ void map_field_to_quadrature_data_tensor_product_2d( } } + +template +MFEM_HOST_DEVICE inline +void map_field_to_quadrature_data_tensor_product_1d( + DeviceTensor<2> &field_qp, + const DofToQuadMap &dtq, + const DeviceTensor<1> &field_e, + const field_operator_t &input, + const DeviceTensor<1, const real_t> &integration_weights, + const std::array, 6> &scratch_mem) +{ + [[maybe_unused]] auto B = dtq.B; + [[maybe_unused]] auto G = dtq.G; + + if constexpr (is_value_fop>::value) + { + auto [q1d, unused, d1d] = B.GetShape(); + const int vdim = input.vdim; + const auto field = Reshape(&field_e[0], d1d, vdim); + auto fqp = Reshape(&field_qp[0], vdim, q1d); + + for (int vd = 0; vd < vdim; vd++) + { + MFEM_FOREACH_THREAD(qx, x, q1d) + { + real_t acc = 0.0; + for (int dx = 0; dx < d1d; dx++) + { + acc += B(qx, 0, dx) * field(dx, vd); + } + fqp(vd, qx) = acc; + } + } + MFEM_SYNC_THREAD; + } + else if constexpr ( + is_gradient_fop>::value) + { + const auto [q1d, unused, d1d] = B.GetShape(); + const int vdim = input.vdim; + const int dim = input.dim; + const auto field = Reshape(&field_e[0], d1d, vdim); + auto fqp = Reshape(&field_qp[0], vdim, dim, q1d); + + for (int vd = 0; vd < vdim; vd++) + { + MFEM_FOREACH_THREAD(qx, x, q1d) + { + real_t acc = 0.0; + for (int dx = 0; dx < d1d; dx++) + { + acc += G(qx, 0, dx) * field(dx, vd); + } + fqp(vd, 0, qx) = acc; + } + MFEM_SYNC_THREAD; + } + } + // TODO: Create separate function for clarity + else if constexpr ( + std::is_same_v, Weight>) + { + const int num_qp = integration_weights.GetShape()[0]; + // TODO: eeek + const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5); + auto w = Reshape(&integration_weights[0], q1d); + auto f = Reshape(&field_qp[0], q1d); + MFEM_FOREACH_THREAD(qx, x, q1d) + { + f(qx) = w(qx); + } + MFEM_SYNC_THREAD; + } + else if constexpr (is_identity_fop>::value) + { + const int q1d = B.GetShape()[0]; + auto field = Reshape(&field_e[0], input.size_on_qp, q1d); + field_qp = field; + } + else + { + static_assert(dfem::always_false>, + "can't map field to quadrature data"); + } +} + template MFEM_HOST_DEVICE void map_field_to_quadrature_data( @@ -444,7 +530,13 @@ void map_fields_to_quadrature_data( if (use_sum_factorization) { - if (dimension == 2) + if (dimension == 1) + { + map_field_to_quadrature_data_tensor_product_1d( + fields_qp[i], dtqmaps[i], field_e, get(fops), + integration_weights, scratch_mem); + } + else if (dimension == 2) { map_field_to_quadrature_data_tensor_product_2d( fields_qp[i], dtqmaps[i], field_e, get(fops), @@ -489,14 +581,20 @@ void map_field_to_quadrature_data_conditional( { if (use_sum_factorization) { - if (dimension == 2) + if (dimension == 1) { - map_field_to_quadrature_data_tensor_product_3d( + map_field_to_quadrature_data_tensor_product_1d( + field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem); + + } + else if (dimension == 2) + { + map_field_to_quadrature_data_tensor_product_2d( field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem); } else if (dimension == 3) { - map_field_to_quadrature_data_tensor_product_2d( + map_field_to_quadrature_data_tensor_product_3d( field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem); } } @@ -547,7 +645,13 @@ void map_direction_to_quadrature_data_conditional( { if (use_sum_factorization) { - if (dimension == 2) + if (dimension == 1) + { + map_field_to_quadrature_data_tensor_product_1d( + directions_qp[i], dtqmaps[i], direction_e, get(fops), + integration_weights, scratch_mem); + } + else if (dimension == 2) { map_field_to_quadrature_data_tensor_product_2d( directions_qp[i], dtqmaps[i], direction_e, get(fops), diff --git a/fem/dfem/qfunction_apply.hpp b/fem/dfem/qfunction_apply.hpp index 0e81f821b7..b420df4406 100644 --- a/fem/dfem/qfunction_apply.hpp +++ b/fem/dfem/qfunction_apply.hpp @@ -44,7 +44,16 @@ void call_qfunction( { if (use_sum_factorization) { - if (dimension == 2) + if (dimension == 1) + { + MFEM_FOREACH_THREAD(q, x, q1d) + { + auto qf_args = decay_tuple {}; + auto r = Reshape(&residual_shmem(0, q), rs_qp); + apply_kernel(r, qfunc, qf_args, input_shmem, q); + } + } + else if (dimension == 2) { MFEM_FOREACH_THREAD(qx, x, q1d) { @@ -123,7 +132,22 @@ void call_qfunction_derivative_action( { if (use_sum_factorization) { - if (dimension == 2) + if (dimension == 1) + { + MFEM_FOREACH_THREAD(q, x, q1d) + { + auto r = Reshape(&residual_shmem(0, q), das_qp); + auto qf_args = decay_tuple {}; +#ifdef MFEM_USE_ENZYME + auto qf_shadow_args = decay_tuple {}; + apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem, + shadow_shmem, q); +#else + apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q); +#endif + } + } + else if (dimension == 2) { MFEM_FOREACH_THREAD(qx, x, q1d) { @@ -164,7 +188,10 @@ void call_qfunction_derivative_action( } } } - MFEM_SYNC_THREAD; + else + { + MFEM_ABORT_KERNEL("unsupported dimension"); + } } else { @@ -180,8 +207,8 @@ void call_qfunction_derivative_action( apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q); #endif } - MFEM_SYNC_THREAD; } + MFEM_SYNC_THREAD; } template diff --git a/fem/dfem/qfunction_transform.hpp b/fem/dfem/qfunction_transform.hpp index 9e4dc7e86b..b0c0cca5a6 100644 --- a/fem/dfem/qfunction_transform.hpp +++ b/fem/dfem/qfunction_transform.hpp @@ -44,7 +44,7 @@ void process_qf_arg( { for (int j = 0; j < n; j++) { - arg(j, i).value = u((i * m) + j); + arg(j, i).value = u((i * n) + j); } } } @@ -94,8 +94,8 @@ void process_qf_arg( { for (int j = 0; j < n; j++) { - arg(j, i).value = u((i * m) + j); - arg(j, i).gradient = v((i * m) + j); + arg(j, i).value = u((i * n) + j); + arg(j, i).gradient = v((i * n) + j); } } } @@ -181,6 +181,14 @@ void process_derivative_from_native_dual( } } +template +MFEM_HOST_DEVICE inline +void process_derivative_from_native_dual( + DeviceTensor<1, T> &r, + const dual &x) +{ + r(0) = x.gradient; +} template MFEM_HOST_DEVICE inline @@ -230,7 +238,7 @@ void process_qf_arg( { for (int j = 0; j < n; j++) { - arg(j, i) = u((i * m) + j); + arg(j, i) = u((i * n) + j); } } } @@ -330,7 +338,7 @@ void process_qf_arg( { for (int j = 0; j < n; j++) { - arg(j, i) = u((i * m) + j); + arg(j, i) = u((i * n) + j); } } } diff --git a/fem/dfem/tuple.hpp b/fem/dfem/tuple.hpp index 8d6832f93c..4035e5fd8b 100644 --- a/fem/dfem/tuple.hpp +++ b/fem/dfem/tuple.hpp @@ -454,7 +454,7 @@ MFEM_HOST_DEVICE constexpr auto operator+=(tuple& x, * * @tparam T the types stored in the tuples x and y * @tparam i integer sequence used to index the tuples - * @param x tuple of values to be subracted from + * @param x tuple of values to be subtracted from * @param y tuple of values to subtract from x */ template @@ -596,7 +596,7 @@ MFEM_HOST_DEVICE constexpr auto div_helper(const real_t a, * @tparam T the types stored in the tuple y * @tparam i The integer sequence to i * @param x tuple of values - * @param a the constant denomenator + * @param a the constant denominator * @return the returned tuple ratio */ template @@ -726,7 +726,7 @@ MFEM_HOST_DEVICE constexpr auto operator*(const tuple& x, const real_t a) /** * @tparam T the types stored in the tuple - * @tparam i a list of indices used to acces each element of the tuple + * @tparam i a list of indices used to access each element of the tuple * @param out the ostream to write the output to * @param A the tuple of values * @brief helper used to implement printing a tuple of values diff --git a/fem/dfem/util.hpp b/fem/dfem/util.hpp index 21db62d9d1..5a9a6c38c8 100644 --- a/fem/dfem/util.hpp +++ b/fem/dfem/util.hpp @@ -327,8 +327,8 @@ void print_mpi_sync(const std::string& msg) // First gather string lengths size_t msg_len = msg.length(); std::vector lengths(nranks); - MPI_Gather(&msg_len, 1, MPI_INT, - lengths.data(), 1, MPI_INT, + MPI_Gather(&msg_len, 1, MPITypeMap::mpi_type, + lengths.data(), 1, MPITypeMap::mpi_type, 0, MPI_COMM_WORLD); if (myrank == 0) @@ -944,7 +944,44 @@ const Operator *get_element_restriction(const FieldDescriptor &f, } else { - static_assert(dfem::always_false, "can't use GetElementRestriction on type"); + static_assert(dfem::always_false, + "can't use get_element_restriction on type"); + } + return nullptr; // Unreachable, but avoids compiler warning + }, f.data); +} + +/// @brief Get the face restriction operator for a field descriptor. +/// +/// @param f the field descriptor. +/// @param o the face dof ordering. +/// @param ft the face type +/// @param m indicator if single or double valued +/// @returns the face restriction operator for the field descriptor in +/// specified ordering. +inline +const Operator *get_face_restriction(const FieldDescriptor &f, + ElementDofOrdering o, + FaceType ft, + L2FaceValues m) +{ + return std::visit([&o, &ft, &m](auto&& arg) -> const Operator* + { + using T = std::decay_t; + if constexpr (std::is_same_v || + std::is_same_v) + { + return arg->GetFaceRestriction(o, ft, m); + } + else if constexpr (std::is_same_v) + { + // ParameterSpace does not support face restrictions + MFEM_ABORT("internal error"); + } + else + { + static_assert(dfem::always_false, + "can't use get_face_restriction on type"); } return nullptr; // Unreachable, but avoids compiler warning }, f.data); @@ -965,6 +1002,11 @@ const Operator *get_restriction(const FieldDescriptor &f, { return get_element_restriction(f, o); } + else if constexpr (std::is_same_v) + { + return get_face_restriction(f, o, FaceType::Boundary, + L2FaceValues::SingleValued); + } MFEM_ABORT("restriction not implemented for Entity"); return nullptr; } @@ -974,7 +1016,7 @@ const Operator *get_restriction(const FieldDescriptor &f, /// @param f the field descriptor. /// @param o the element dof ordering. /// @param fop the field operator. -/// @returns a tuple containting a std::function with the transpose +/// @returns a tuple containing a std::function with the transpose /// restriction callback and it's height. template inline std::tuple, int> @@ -1076,6 +1118,24 @@ void prolongation(const std::vector fields, } } +inline +void get_lvectors(const std::vector fields, + const Vector &x, + std::vector &fields_l) +{ + int data_offset = 0; + for (std::size_t i = 0; i < fields.size(); i++) + { + const int sz = GetVSize(fields[i]); + fields_l[i].SetSize(sz); + + const Vector x_i(const_cast(x), data_offset, sz); + fields_l[i] = x_i; + + data_offset += sz; + } +} + /// @brief Get a transpose prolongation callback for a field descriptor. /// /// In the special case of a one field operator, the transpose prolongation @@ -1371,7 +1431,7 @@ create_descriptors_to_fields_map( if constexpr (std::is_same_v, Weight>) { // TODO-bug: stealing dimension from the first field - fop.dim = GetDimension(fields[0]); + fop.dim = GetDimension(fields[0]); fop.vdim = 1; fop.size_on_qp = 1; map = -1; diff --git a/fem/fe/face_map_utils.hpp b/fem/fe/face_map_utils.hpp index b3e6a225d1..ac921811ba 100644 --- a/fem/fe/face_map_utils.hpp +++ b/fem/fe/face_map_utils.hpp @@ -259,6 +259,30 @@ inline void FaceIdxToVolIdx3D(const int index, const int size1d, i = yz_plane ? level : _i; } +MFEM_HOST_DEVICE +inline int FaceIdxToVolIdx(int dim, int i, int size1d, int face0, int face1, + int side, int orientation) +{ + if (dim == 2) + { + int ix, iy; + internal::FaceIdxToVolIdx2D(i, size1d, face0, face1, side, ix, iy); + return ix + iy*size1d; + } + else if (dim == 3) + { + int ix, iy, iz; + internal::FaceIdxToVolIdx3D(i, size1d, face0, face1, side, orientation, + ix, iy, iz); + return ix + size1d*iy + size1d*size1d*iz; + } + else + { + MFEM_ABORT_KERNEL("Invalid dimension"); + return -1; + } +}; + } // namespace internal } // namespace mfem diff --git a/fem/fe/fe_base.cpp b/fem/fe/fe_base.cpp index ab1977ddbe..fb51d44181 100644 --- a/fem/fe/fe_base.cpp +++ b/fem/fe/fe_base.cpp @@ -661,65 +661,78 @@ void ScalarFiniteElement::ScalarLocalL2Restriction( void NodalFiniteElement::CreateLexicographicFullMap(const IntegrationRule &ir) const { - // Get the FULL version of the map. - auto &d2q = GetDofToQuad(ir, DofToQuad::FULL); - //Undo the native ordering which is what FiniteElement::GetDofToQuad returns. - auto *d2q_new = new DofToQuad(d2q); - d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL; - const int nqpt = ir.GetNPoints(); - const int b_dim = (range_type == VECTOR) ? dim : 1; - - for (int i = 0; i < nqpt; i++) +#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP) + #pragma omp critical (DofToQuad) +#endif { - for (int d = 0; d < b_dim; d++) + // Get the FULL version of the map. + auto &d2q = GetDofToQuad(ir, DofToQuad::FULL); + //Undo the native ordering which is what FiniteElement::GetDofToQuad returns. + auto *d2q_new = new DofToQuad(d2q); + d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL; + const int nqpt = ir.GetNPoints(); + + const int b_dim = (range_type == VECTOR) ? dim : 1; + + for (int i = 0; i < nqpt; i++) { - for (int j = 0; j < dof; j++) + for (int d = 0; d < b_dim; d++) { - const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])]; - d2q_new->B[i+nqpt*(d+b_dim*j)] = val; - d2q_new->Bt[j+dof*(i+nqpt*d)] = val; + for (int j = 0; j < dof; j++) + { + const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])]; + d2q_new->B[i+nqpt*(d+b_dim*j)] = val; + d2q_new->Bt[j+dof*(i+nqpt*d)] = val; + } } } - } - const int g_dim = [this]() - { - switch (deriv_type) + const int g_dim = [this]() { - case GRAD: return dim; - case DIV: return 1; - case CURL: return cdim; - default: return 0; - } - }(); - - for (int i = 0; i < nqpt; i++) - { - for (int d = 0; d < g_dim; d++) - { - for (int j = 0; j < dof; j++) + switch (deriv_type) { - const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])]; - d2q_new->G[i+nqpt*(d+g_dim*j)] = val; - d2q_new->Gt[j+dof*(i+nqpt*d)] = val; + case GRAD: return dim; + case DIV: return 1; + case CURL: return cdim; + default: return 0; + } + }(); + + for (int i = 0; i < nqpt; i++) + { + for (int d = 0; d < g_dim; d++) + { + for (int j = 0; j < dof; j++) + { + const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])]; + d2q_new->G[i+nqpt*(d+g_dim*j)] = val; + d2q_new->Gt[j+dof*(i+nqpt*d)] = val; + } } } - } - dof2quad_array.Append(d2q_new); + dof2quad_array.Append(d2q_new); + } } const DofToQuad &NodalFiniteElement::GetDofToQuad(const IntegrationRule &ir, DofToQuad::Mode mode) const { - //Should make this loop a function of FiniteElement - for (int i = 0; i < dof2quad_array.Size(); i++) + DofToQuad *d2q = nullptr; +#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP) + #pragma omp critical (DofToQuad) +#endif { - const DofToQuad &d2q = *dof2quad_array[i]; - if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; } + //Should make this loop a function of FiniteElement + for (int i = 0; i < dof2quad_array.Size(); i++) + { + d2q = dof2quad_array[i]; + if (d2q->IntRule == &ir && d2q->mode == mode) { break; } + d2q = nullptr; + } } - + if (d2q) { return *d2q; } if (mode != DofToQuad::LEXICOGRAPHIC_FULL) { return FiniteElement::GetDofToQuad(ir, mode); diff --git a/fem/fe_coll.cpp b/fem/fe_coll.cpp index 0d7f18dd12..a5cf054a99 100644 --- a/fem/fe_coll.cpp +++ b/fem/fe_coll.cpp @@ -308,13 +308,25 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name) FiniteElement::INTEGRAL, BasisType::GetType(name[12])); } - else if (!strncmp(name, "RT_R1D",6)) + else if (!strncmp(name, "RT_R1D_", 7)) { - fec = new RT_R1D_FECollection(atoi(name+11),atoi(name + 7)); + fec = new RT_R1D_FECollection(atoi(name + 11), atoi(name + 7)); } - else if (!strncmp(name, "RT_R2D",6)) + else if (!strncmp(name, "RT_R1D@", 7)) { - fec = new RT_R2D_FECollection(atoi(name+11),atoi(name + 7)); + fec = new RT_R1D_FECollection(atoi(name + 14), atoi(name + 10), + BasisType::GetType(name[7]), + BasisType::GetType(name[8])); + } + else if (!strncmp(name, "RT_R2D_", 7)) + { + fec = new RT_R2D_FECollection(atoi(name + 11), atoi(name + 7)); + } + else if (!strncmp(name, "RT_R2D@", 7)) + { + fec = new RT_R2D_FECollection(atoi(name + 14), atoi(name + 10), + BasisType::GetType(name[7]), + BasisType::GetType(name[8])); } else if (!strncmp(name, "RT_", 3)) { @@ -336,13 +348,25 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name) BasisType::GetType(name[9]), BasisType::GetType(name[10])); } - else if (!strncmp(name, "ND_R1D",6)) + else if (!strncmp(name, "ND_R1D_", 7)) { - fec = new ND_R1D_FECollection(atoi(name+11),atoi(name + 7)); + fec = new ND_R1D_FECollection(atoi(name + 11), atoi(name + 7)); } - else if (!strncmp(name, "ND_R2D",6)) + else if (!strncmp(name, "ND_R1D@", 7)) { - fec = new ND_R2D_FECollection(atoi(name+11),atoi(name + 7)); + fec = new ND_R1D_FECollection(atoi(name + 14), atoi(name + 10), + BasisType::GetType(name[7]), + BasisType::GetType(name[8])); + } + else if (!strncmp(name, "ND_R2D_", 7)) + { + fec = new ND_R2D_FECollection(atoi(name + 11), atoi(name + 7)); + } + else if (!strncmp(name, "ND_R2D@", 7)) + { + fec = new ND_R2D_FECollection(atoi(name + 14), atoi(name + 10), + BasisType::GetType(name[7]), + BasisType::GetType(name[8])); } else if (!strncmp(name, "ND_", 3)) { @@ -401,9 +425,6 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name) { MFEM_ABORT("unknown FiniteElementCollection: " << name); } - MFEM_VERIFY(!strcmp(fec->Name(), name), "input name: \"" << name - << "\" does not match the created collection name: \"" - << fec->Name() << '"'); return fec; } @@ -2459,8 +2480,7 @@ RT_FECollection::RT_FECollection(const int order, const int dim, const char *cb_name = BasisType::Name(cb_type); // this may abort MFEM_ABORT("unknown closed BasisType: " << cb_name); } - if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid && - ob_type != BasisType::IntegratedGLL) + if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid) { const char *ob_name = BasisType::Name(ob_type); // this may abort MFEM_ABORT("unknown open BasisType: " << ob_name); @@ -2518,6 +2538,7 @@ RT_FECollection::RT_FECollection(const int p, const int dim, const int map_type, const bool signs, const int ob_type) : FiniteElementCollection(p + 1) + , dim(dim) , ob_type(ob_type) { if (Quadrature1D::CheckOpen(BasisType::GetQuadrature1D(ob_type)) == @@ -2786,8 +2807,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim, int cp_type = BasisType::GetQuadrature1D(cb_type); // Error checking - if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid && - ob_type != BasisType::IntegratedGLL) + if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid) { const char *ob_name = BasisType::Name(ob_type); MFEM_ABORT("Invalid open basis point type: " << ob_name); diff --git a/fem/fe_coll.hpp b/fem/fe_coll.hpp index 8b8c2729e4..0206da7ef9 100644 --- a/fem/fe_coll.hpp +++ b/fem/fe_coll.hpp @@ -120,7 +120,9 @@ public: | ND_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) | | ND_Trace@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) | | ND_R1D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. | + | ND_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. | | ND_R2D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. | + | ND_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. | | RT_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | Raviart-Thomas vector elements | | RT@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | Raviart-Thomas vector elements | | RT_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) | @@ -128,7 +130,9 @@ public: | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) | | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) | | RT_R1D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. | + | RT_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. | | RT_R2D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. | + | RT_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. | | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements | | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements | | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements | @@ -464,6 +468,13 @@ public: RT_Trace_FECollection(const int p, const int dim, const int map_type = FiniteElement::INTEGRAL, const int ob_type = BasisType::GaussLegendre); + + FiniteElementCollection *Clone(int p) const override + { + const int map_type = (strncmp(rt_name, "RT_Trace", 8) == 0)? + (FiniteElement::INTEGRAL):(FiniteElement::VALUE); + return new RT_Trace_FECollection(p, dim, map_type, ob_type); + } }; /** Arbitrary order discontinuous finite elements defined on the interface @@ -475,6 +486,13 @@ public: DG_Interface_FECollection(const int p, const int dim, const int map_type = FiniteElement::VALUE, const int ob_type = BasisType::GaussLegendre); + + FiniteElementCollection *Clone(int p) const override + { + const int map_type = (strncmp(rt_name, "DG_Iface", 8) == 0)? + (FiniteElement::VALUE):(FiniteElement::INTEGRAL); + return new DG_Interface_FECollection(p, dim, map_type, ob_type); + } }; /// Arbitrary order H(curl)-conforming Nedelec finite elements. diff --git a/fem/fes_kernels.hpp b/fem/fes_kernels.hpp index 0691da29f7..f3da82d840 100644 --- a/fem/fes_kernels.hpp +++ b/fem/fes_kernels.hpp @@ -224,7 +224,7 @@ struct DerefineMatrixOpMultFunctor sum += sign * bsptr[boptr[k] + i + j * block_height] * xptr[this->IndexX(col, vdim, k)]; } -#if defined(__CUDA_ARCH__) or defined(__HIP_DEVICE_COMPILE__) +#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) if (Atomic) { atomicAdd(yptr + this->IndexY(row, vdim), sum); diff --git a/fem/fespace.hpp b/fem/fespace.hpp index 0265f33d1c..0e7f508980 100644 --- a/fem/fespace.hpp +++ b/fem/fespace.hpp @@ -683,8 +683,12 @@ public: NURBSExtension *GetNURBSext() { return NURBSext; } NURBSExtension *StealNURBSext(); - bool Conforming() const { return mesh->Conforming() && cP == NULL; } - bool Nonconforming() const { return mesh->Nonconforming() || cP != NULL; } + bool Conforming() const + { + return NURBSext != NULL || + (mesh->Conforming() && cP == NULL); + } + bool Nonconforming() const { return !Conforming(); } /** Set the prolongation operator of the space to an arbitrary sparse matrix, creating a copy of the argument. */ @@ -922,6 +926,9 @@ public: { return mesh->GetBdrElementType(i); } /// Returns ElementTransformation for the @a i-th element. + /// @note The returned pointer references an object owned by the associated + /// @a Mesh that will be modified by other calls to `GetElementTransformation`. + /// As such, this pointer should @b not be deleted by the caller. ElementTransformation *GetElementTransformation(int i) const { return mesh->GetElementTransformation(i); } diff --git a/fem/gridfunc.cpp b/fem/gridfunc.cpp index a789815c59..062d4b4c0d 100644 --- a/fem/gridfunc.cpp +++ b/fem/gridfunc.cpp @@ -68,7 +68,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input) Vector::Load(input, fes->GetVSize()); // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering - if (fes->Nonconforming() && + if (fes->Nonconforming() && fes->GetMesh()->ncmesh && fes->GetMesh()->ncmesh->IsLegacyLoaded()) { LegacyNCReorder(); @@ -1374,6 +1374,50 @@ void GridFunction::GetVectorGradientHat( MultAtB(loc_data_mat, dshape, gh); } +void GridFunction::GetGradients(const IntegrationRule &ir, Vector &grad, + QVectorLayout ql, MemoryType d_mt) const +{ + const FiniteElement &fe = *fes->GetTypicalFE(); + const int dim = fe.GetDim(); + const int vdim = fes->GetVDim(); + const int NE = fes->GetNE(); + const int ND = fe.GetDof(); + const int NQ = ir.GetNPoints(); + + MemoryType my_d_mt = (d_mt != MemoryType::DEFAULT) ? d_mt : + Device::GetDeviceMemoryType(); + + // ql == QVectorLayout::byNODES : NQ x VDIM x DIM x NE + // ql == QVectorLayout::byVDIM : VDIM x DIM x NQPT x NE + grad.SetSize(dim*vdim*NQ*NE, my_d_mt); + + const QuadratureInterpolator &qi = *fes->GetQuadratureInterpolator(ir); + qi.SetOutputLayout(ql); + + const bool use_tensor_products = UsesTensorBasis(*fes); + qi.DisableTensorProducts(!use_tensor_products); + const ElementDofOrdering e_ordering = use_tensor_products ? + ElementDofOrdering::LEXICOGRAPHIC : + ElementDofOrdering::NATIVE; + const Operator *elem_restr = fes->GetElementRestriction(e_ordering); + + // Pre-compute the geometric factors in order to set the desired MemoryType + // they use: + fes->GetMesh()->GetGeometricFactors( + ir, GeometricFactors::JACOBIANS, my_d_mt); + + if (elem_restr) // currently, always true + { + Vector f_e(vdim*ND*NE, my_d_mt); + elem_restr->Mult(*this, f_e); + qi.PhysDerivatives(f_e, grad); + } + else + { + qi.PhysDerivatives(*this, grad); + } +} + real_t GridFunction::GetDivergence(ElementTransformation &T) const { DofTransformation doftrans; @@ -2624,7 +2668,7 @@ void GridFunction::ProjectBdrCoefficient(Coefficient *coeff[], } for (int i = 0; i < values_counter.Size(); i++) { - MFEM_ASSERT(bool(values_counter[i]) == ess_vdofs_marker[i], + MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]), "internal error"); } #endif diff --git a/fem/gridfunc.hpp b/fem/gridfunc.hpp index c3ecb5b1ee..c64ad4a528 100644 --- a/fem/gridfunc.hpp +++ b/fem/gridfunc.hpp @@ -153,7 +153,8 @@ public: /// Shortcut for calling SetFromTrueDofs() with GetTrueVector() as argument. void SetFromTrueVector() { SetFromTrueDofs(GetTrueVector()); } - /// Returns the values in the vertices of i'th element for dimension vdim. + /** @brief Returns the values at the vertices of element @a i for the 1-based + dimension vdim. */ void GetNodalValues(int i, Array &nval, int vdim = 1) const; /** @name Element index Get Value Methods @@ -308,7 +309,8 @@ public: /// For a vector grid function, makes sure that the ordering is byNODES. void ReorderByNodes(); - /// Return the values as a vector on mesh vertices for dimension vdim. + /** @brief Returns the values as a vector at mesh vertices, for the 1-based + dimension vdim. */ void GetNodalValues(Vector &nval, int vdim = 1) const; void GetVectorFieldNodalValues(Vector &val, int comp) const; @@ -359,6 +361,33 @@ public: variable. */ void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const; + /** @brief Evaluate the gradients of the GridFunction at the given quadrature + points, @a ir, in all mesh elements. */ + /** This method assumes that all mesh elements are the same type and that the + IntegrationRule @a ir is consistent with that type of element. + + @param[in] ir Quadrature points at which the gradients are to be + evaluated. + @param[out] grad Output vector of size `SDIM*VDIM*NQ*NE` where `SDIM` is + the spatial dimention of the mesh, `VDIM` is the vector + dimension of the GridFunction, `NQ` is the number of + quadrature points in @a ir, and `NE` is the number of + elements in the mesh. The layout of @a grad is + determined by the parameter @a ql: when @a ql is + QVectorLayout::byNODES, the layout is + `NQ x VDIM x SDIM x NE`; when @a ql is + QVectorLayout::byVDIM, the layout is + `VDIM x SDIM x NQ x NE`. + @param[in] ql Determines the layout of the output vector @a grad; see + the description of @a grad for details. + @param[in] d_mt MemoryType to use for allocating the output vector + @a grad, as well the GeometricFactors and temporary + vector used by the method. By default, the current + device memory type is used. */ + void GetGradients(const IntegrationRule &ir, Vector &grad, + QVectorLayout ql = QVectorLayout::byNODES, + MemoryType d_mt = MemoryType::DEFAULT) const; + /** Compute $ (\int_{\Omega} (*this) \psi_i)/(\int_{\Omega} \psi_i) $, where $ \psi_i $ are the basis functions for the FE space of avgs. Both FE spaces should be scalar and on the same mesh. */ diff --git a/fem/gslib.cpp b/fem/gslib.cpp index 1106d873d2..520c2fff6b 100644 --- a/fem/gslib.cpp +++ b/fem/gslib.cpp @@ -99,10 +99,10 @@ FindPointsGSLIB::FindPointsGSLIB() gf_rst_map.SetSize(4); for (int i = 0; i < mesh_split.Size(); i++) { - mesh_split[i] = NULL; - ir_split[i] = NULL; - fes_rst_map[i] = NULL; - gf_rst_map[i] = NULL; + mesh_split[i] = nullptr; + ir_split[i] = nullptr; + fes_rst_map[i] = nullptr; + gf_rst_map[i] = nullptr; } gsl_comm = new gslib::comm; @@ -119,20 +119,33 @@ FindPointsGSLIB::FindPointsGSLIB() crystal_init(cr, gsl_comm); } +FindPointsGSLIB::FindPointsGSLIB(Mesh &mesh_in, const double bb_t, + const double newt_tol, const int npt_max) + : FindPointsGSLIB() +{ + Setup(mesh_in, bb_t, newt_tol, npt_max); +} + FindPointsGSLIB::~FindPointsGSLIB() { - crystal_free(cr); - comm_free(gsl_comm); - delete gsl_comm; - delete cr; - for (int i = 0; i < 4; i++) + FreeData(); +#ifdef MFEM_USE_MPI + if (!Mpi::IsFinalized()) // currently segfaults inside gslib otherwise +#endif { - if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = NULL; } - if (ir_split[i]) { delete ir_split[i]; ir_split[i] = NULL; } - if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = NULL; } - if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = NULL; } + crystal_free(cr); + comm_free(gsl_comm); + delete gsl_comm; + delete cr; } - if (fec_map_lin) { delete fec_map_lin; fec_map_lin = NULL; } + for (int i = 0; i < mesh_split.Size(); i++) + { + if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = nullptr; } + if (ir_split[i]) { delete ir_split[i]; ir_split[i] = nullptr; } + if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = nullptr; } + if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = nullptr; } + } + if (fec_map_lin) { delete fec_map_lin; fec_map_lin = nullptr; } } #ifdef MFEM_USE_MPI @@ -150,10 +163,10 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_) gf_rst_map.SetSize(4); for (int i = 0; i < mesh_split.Size(); i++) { - mesh_split[i] = NULL; - ir_split[i] = NULL; - fes_rst_map[i] = NULL; - gf_rst_map[i] = NULL; + mesh_split[i] = nullptr; + ir_split[i] = nullptr; + fes_rst_map[i] = nullptr; + gf_rst_map[i] = nullptr; } gsl_comm = new gslib::comm; @@ -161,6 +174,13 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_) comm_init(gsl_comm, comm_); crystal_init(cr, gsl_comm); } + +FindPointsGSLIB::FindPointsGSLIB(ParMesh &mesh_in, const double bb_t, + const double newt_tol, const int npt_max) + : FindPointsGSLIB(mesh_in.GetComm()) +{ + Setup(mesh_in, bb_t, newt_tol, npt_max); +} #endif void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol, @@ -180,37 +200,9 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol, mesh = &m; dim = mesh->Dimension(); spacedim = dim; - unsigned dof1D = meshOrder + 1; + const unsigned dof1D = meshOrder + 1; - SetupSplitMeshes(); - if (dim == 2) - { - if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } - ir_split[0] = new IntegrationRule(3*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder); - - if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; } - ir_split[1] = new IntegrationRule(pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder); - } - else if (dim == 3) - { - if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } - ir_split[0] = new IntegrationRule(pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder); - - if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; } - ir_split[1] = new IntegrationRule(4*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder); - - if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; } - ir_split[2] = new IntegrationRule(3*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder); - - if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; } - ir_split[3] = new IntegrationRule(8*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder); - } + SetupSplitMeshesAndIntegrationRules(meshOrder); GetNodalValues(mesh->GetNodes(), gsl_mesh); @@ -870,6 +862,7 @@ void FindPointsGSLIB::SetupSurf(Mesh &m, const double bb_t, mesh = &m; dim = mesh->Dimension(); // This is reference dimension spacedim = mesh->SpaceDimension(); // This is physical dimension + MFEM_VERIFY(dim < 3, "Configuration not supported yet."); bool tensor_product_only = mesh->GetNE() == 0 || (mesh->GetNumGeometries(dim) == 1 && @@ -884,27 +877,7 @@ void FindPointsGSLIB::SetupSurf(Mesh &m, const double bb_t, const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder(); unsigned dof1D = meshOrder + 1; - SetupSplitMeshes(); - if (dim == 1) - { - if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } - ir_split[0] = new IntegrationRule(pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder); - } - else if (dim == 2) - { - if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } - ir_split[0] = new IntegrationRule(3*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder); - - if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; } - ir_split[1] = new IntegrationRule(pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder); - } - else - { - MFEM_ABORT("3D surface meshes not supported"); - } + SetupSplitMeshesAndIntegrationRules(meshOrder); GetNodalValues(mesh->GetNodes(), gsl_mesh); @@ -2612,15 +2585,22 @@ void FindPointsGSLIB::FreeData() if (!setupflag) { return; } if (dim == spacedim) { - if (dim == 2) +#ifdef MFEM_USE_MPI + if (!Mpi::IsFinalized()) // currently segfaults inside gslib otherwise +#endif { - findpts_free_2((gslib::findpts_data_2 *)this->fdataD); - } - else - { - findpts_free_3((gslib::findpts_data_3 *)this->fdataD); + if (dim == 2) + { + findpts_free_2((gslib::findpts_data_2 *)this->fdataD); + } + else + { + findpts_free_3((gslib::findpts_data_3 *)this->fdataD); + } } +#ifdef MFEM_USE_MPI } +#endif gsl_code.DeleteAll(); gsl_proc.DeleteAll(); gsl_elem.DeleteAll(); @@ -2643,7 +2623,7 @@ void FindPointsGSLIB::FreeData() void FindPointsGSLIB::SetupSplitMeshes() { - fec_map_lin = new H1_FECollection(1, dim); + if (fec_map_lin == nullptr) { fec_map_lin = new H1_FECollection(1, dim); } if (mesh->Dimension() == 1) { mesh_split[0] = new Mesh(Mesh::MakeCartesian1D(1, Element::SEGMENT)); @@ -2690,7 +2670,7 @@ void FindPointsGSLIB::SetupSplitMeshes() mesh_split[1] = new Mesh(Mesh::MakeCartesian2D(1, 1, Element::QUADRILATERAL)); } - else if (mesh->Dimension() == 3) + else if (dim == 3) { mesh_split[0] = new Mesh(Mesh::MakeCartesian3D(1, 1, 1, Element::HEXAHEDRON)); @@ -2835,42 +2815,6 @@ void FindPointsGSLIB::SetupSplitMeshes() } } } - - NE_split_total = 0; - split_element_map.SetSize(0); - split_element_index.SetSize(0); - int NEsplit = 0; - for (int e = 0; e < mesh->GetNE(); e++) - { - const Geometry::Type gt = mesh->GetElement(e)->GetGeometryType(); - if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM) - { - NEsplit = 3; - } - else if (gt == Geometry::TETRAHEDRON) - { - NEsplit = 4; - } - else if (gt == Geometry::PYRAMID) - { - NEsplit = 8; - } - else if (gt == Geometry::SEGMENT || gt == Geometry::SQUARE || - gt == Geometry::CUBE) - { - NEsplit = 1; - } - else - { - MFEM_ABORT("Unsupported geometry type."); - } - NE_split_total += NEsplit; - for (int i = 0; i < NEsplit; i++) - { - split_element_map.Append(e); - split_element_index.Append(i); - } - } } void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin, @@ -2924,6 +2868,87 @@ void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin, } } +void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order) +{ + MFEM_VERIFY(mesh, "Setup FindPointsGSLIB with mesh first."); + const int dof1D = order+1; + const int dim = mesh->Dimension(); + + SetupSplitMeshes(); + if (dim == 1) + { + if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } + ir_split[0] = new IntegrationRule(pow(dof1D, dim)); + SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order); + } + else if (dim == 2) + { + if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } + ir_split[0] = new IntegrationRule(3*pow(dof1D, dim)); + SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order); + + if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; } + ir_split[1] = new IntegrationRule(pow(dof1D, dim)); + SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order); + } + else if (dim == 3) + { + if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } + ir_split[0] = new IntegrationRule(pow(dof1D, dim)); + SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order); + + if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; } + ir_split[1] = new IntegrationRule(4*pow(dof1D, dim)); + SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order); + + if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; } + ir_split[2] = new IntegrationRule(3*pow(dof1D, dim)); + SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], order); + + if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; } + ir_split[3] = new IntegrationRule(8*pow(dof1D, dim)); + SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], order); + } + + // Setup map for non tensor-product elements + NE_split_total = 0; + split_element_map.SetSize(0); + split_element_index.SetSize(0); + int NEsplit = 0; + for (int e = 0; e < mesh->GetNE(); e++) + { + const Geometry::Type gt = mesh->GetElement(e)->GetGeometryType(); + if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM) + { + NEsplit = 3; + } + else if (gt == Geometry::TETRAHEDRON) + { + NEsplit = 4; + } + else if (gt == Geometry::PYRAMID) + { + NEsplit = 8; + } + + else if (gt == Geometry::SEGMENT || + gt == Geometry::SQUARE || gt == Geometry::CUBE) + { + NEsplit = 1; + } + else + { + MFEM_ABORT("Unsupported geometry type."); + } + NE_split_total += NEsplit; + for (int i = 0; i < NEsplit; i++) + { + split_element_map.Append(e); + split_element_index.Append(i); + } + } +} + void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in, Vector &node_vals) { @@ -3661,6 +3686,19 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in, } // parallel } +Array FindPointsGSLIB::GetPointsNotFoundIndices() const +{ + Array nf_idxs; + for (int i = 0; i < gsl_code.Size(); i++) + { + if (gsl_code[i] == 2) + { + nf_idxs.Append(i); + } + } + return nf_idxs; +} + void FindPointsGSLIB::DistributePointInfoToOwningMPIRanks( Array &recv_elem, Vector &recv_ref, Array &recv_code) @@ -4129,6 +4167,10 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid, { MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required."); const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder(); + const int gfOrder = gfmax ? gfmax->FESpace()->GetMaxElementOrder() : + meshOrder; + MFEM_VERIFY(meshOrder == gfOrder, + "Mesh order must match gfmax order in OversetFindPointsGSLIB."); // FreeData if OversetFindPointsGSLIB::Setup has been called already if (setupflag) { FreeData(); } @@ -4138,35 +4180,7 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid, const FiniteElement *fe = mesh->GetNodalFESpace()->GetTypicalFE(); unsigned dof1D = fe->GetOrder() + 1; - SetupSplitMeshes(); - if (dim == 2) - { - if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } - ir_split[0] = new IntegrationRule(3*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder); - - if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; } - ir_split[1] = new IntegrationRule(pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder); - } - else if (dim == 3) - { - if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; } - ir_split[0] = new IntegrationRule(pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder); - - if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; } - ir_split[1] = new IntegrationRule(4*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder); - - if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; } - ir_split[2] = new IntegrationRule(3*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder); - - if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; } - ir_split[3] = new IntegrationRule(8*pow(dof1D, dim)); - SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder); - } + SetupSplitMeshesAndIntegrationRules(meshOrder); GetNodalValues(mesh->GetNodes(), gsl_mesh); @@ -4223,7 +4237,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos, { MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before " "finding points."); - MFEM_VERIFY(overset, "Please setup FindPoints for overlapping grids."); + MFEM_VERIFY(overset, "Please use OversetFindPoints for overlapping grids."); points_cnt = point_pos.Size() / dim; unsigned int match = 0; // Don't find points in the mesh if point_id=mesh_id diff --git a/fem/gslib.hpp b/fem/gslib.hpp index 51c54ef36e..f536ad80e6 100644 --- a/fem/gslib.hpp +++ b/fem/gslib.hpp @@ -13,7 +13,11 @@ #define MFEM_GSLIB #include "../config/config.hpp" +#ifdef MFEM_USE_MPI +#include "pgridfunc.hpp" +#else #include "gridfunc.hpp" +#endif #ifdef MFEM_USE_GSLIB @@ -135,6 +139,10 @@ protected: IntegrationRule *irule, int order); + /// Helper function that calls \ref SetupSplitMeshes and + /// \ref SetupIntegrationRuleForSplitMesh. + virtual void SetupSplitMeshesAndIntegrationRules(const int order); + /// Get GridFunction value at the points expected by GSLIB. virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals); @@ -264,12 +272,20 @@ protected: public: FindPointsGSLIB(); + FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1, + const double newt_tol = 1.0e-12, + const int npt_max = 256); #ifdef MFEM_USE_MPI FindPointsGSLIB(MPI_Comm comm_); + FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1, + const double newt_tol = 1.0e-12, + const int npt_max = 256); #endif virtual ~FindPointsGSLIB(); + FindPointsGSLIB(const FindPointsGSLIB&) = delete; + FindPointsGSLIB& operator=(const FindPointsGSLIB&) = delete; /** Initializes the internal mesh in gslib, by sending the positions of the Gauss-Lobatto nodes of the input Mesh object \p m. @@ -284,8 +300,8 @@ public: @param[in] npt_max (Optional) Number of points for simultaneous iteration. This alters performance and memory footprint.*/ - void Setup(Mesh &m, const double bb_t = 0.1, - const double newt_tol = 1.0e-12, + + void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12, const int npt_max = 256); @@ -374,7 +390,12 @@ public: /** Cleans up memory allocated internally by gslib. Note that in parallel, this must be called before MPI_Finalize(), as it - calls MPI_Comm_free() for internal gslib communicators. */ + calls MPI_Comm_free() for internal gslib communicators. FreeData is + also called by the class destructor and there are no memory leaks if the + destructor is called before MPI_Finalize(). If the destructor is called + after MPI_Finalize(), there will be an error because gslib will try to + invoke some MPI functions. + */ virtual void FreeData(); /// Return code for each point searched by FindPoints: inside element (0), on @@ -397,6 +418,9 @@ public: /// point found by FindPoints. virtual const Vector &GetGSLIBReferencePosition() const { return gsl_ref; } + /// Get array of indices of not-found points. + Array GetPointsNotFoundIndices() const; + /** @name Methods to support a custom interpolation procedure. \brief The physical-space point that the user seeks to interpolate at could be located inside an element on another mpi rank. diff --git a/fem/hyperbolic.cpp b/fem/hyperbolic.cpp index d61d8c244f..df773bd31c 100644 --- a/fem/hyperbolic.cpp +++ b/fem/hyperbolic.cpp @@ -181,7 +181,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector( // current elements' the number of degrees of freedom // does not consider the number of equations const int dof1 = el1.GetDof(); - const int dof2 = el2.GetDof(); + const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0); #ifdef MFEM_THREAD_SAFE // Local storage for element integration @@ -219,7 +219,9 @@ void HyperbolicFormIntegrator::AssembleFaceVector( const IntegrationRule *ir = IntRule; if (!ir) { - const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset; + const int max_el_order = dof2 ? std::max(el1.GetOrder(), + el2.GetOrder()) : el1.GetOrder(); + const int order = 2*max_el_order + IntOrderOffset; ir = &IntRules.Get(Tr.GetGeometryType(), order); } // loop over integration points @@ -231,18 +233,22 @@ void HyperbolicFormIntegrator::AssembleFaceVector( // Calculate basis functions on both elements at the face el1.CalcShape(Tr.GetElement1IntPoint(), shape1); - el2.CalcShape(Tr.GetElement2IntPoint(), shape2); // Interpolate elfun at the point elfun1_mat.MultTranspose(shape1, state1); - elfun2_mat.MultTranspose(shape2, state2); + + if (dof2) + { + // Calculate basis functions on both elements at the face + el2.CalcShape(Tr.GetElement2IntPoint(), shape2); + // Interpolate elfun at the point + elfun2_mat.MultTranspose(shape2, state2); + } // Get the normal vector and the flux on the face if (nor.Size() == 1) // if 1D, use 1 or -1. { - // This assume the 1D integration point is in (0,1). This may not work - // if this changes. - nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0; + nor(0) = 2*Tr.GetElement1IntPoint().x - 1.; } else { @@ -250,14 +256,18 @@ void HyperbolicFormIntegrator::AssembleFaceVector( } // Compute F(u+, x) and F(u-, x) with maximum characteristic speed // Compute hat(F) using evaluated quantities - const real_t speed = numFlux.Eval(state1, state2, nor, Tr, fluxN); + const real_t speed = (dof2) ? numFlux.Eval(state1, state2, nor, Tr, fluxN): + fluxFunction.ComputeFluxDotN(state1, nor, Tr, fluxN); // Update the global max char speed max_char_speed = std::max(speed, max_char_speed); // pre-multiply integration weight to flux AddMult_a_VWt(-ip.weight*sign, shape1, fluxN, elvect1_mat); - AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat); + if (dof2) + { + AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat); + } } } @@ -268,7 +278,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad( // current elements' the number of degrees of freedom // does not consider the number of equations const int dof1 = el1.GetDof(); - const int dof2 = el2.GetDof(); + const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0); #ifdef MFEM_THREAD_SAFE // Local storage for element integration @@ -302,7 +312,9 @@ void HyperbolicFormIntegrator::AssembleFaceGrad( const IntegrationRule *ir = IntRule; if (!ir) { - const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset; + const int max_el_order = dof2 ? std::max(el1.GetOrder(), + el2.GetOrder()) : el1.GetOrder(); + const int order = 2*max_el_order + IntOrderOffset; ir = &IntRules.Get(Tr.GetGeometryType(), order); } // loop over integration points @@ -312,20 +324,25 @@ void HyperbolicFormIntegrator::AssembleFaceGrad( Tr.SetAllIntPoints(&ip); // set face and element int. points - // Calculate basis functions on both elements at the face + // Calculate basis functions of the first element at the face el1.CalcShape(Tr.GetElement1IntPoint(), shape1); - el2.CalcShape(Tr.GetElement2IntPoint(), shape2); // Interpolate elfun at the point elfun1_mat.MultTranspose(shape1, state1); - elfun2_mat.MultTranspose(shape2, state2); + + if (dof2) + { + // Calculate basis function of the second element at the face + el2.CalcShape(Tr.GetElement2IntPoint(), shape2); + + // Interpolate elfun at the point + elfun2_mat.MultTranspose(shape2, state2); + } // Get the normal vector and the flux on the face if (nor.Size() == 1) // if 1D, use 1 or -1. { - // This assume the 1D integration point is in (0,1). This may not work - // if this changes. - nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0; + nor(0) = 2*Tr.GetElement1IntPoint().x - 1.; } else { @@ -335,7 +352,14 @@ void HyperbolicFormIntegrator::AssembleFaceGrad( // Trial side 1 // Compute hat(J) using evaluated quantities - numFlux.Grad(1, state1, state2, nor, Tr, JDotN); + if (dof2) + { + numFlux.Grad(1, state1, state2, nor, Tr, JDotN); + } + else + { + fluxFunction.ComputeFluxJacobianDotN(state1, nor, Tr, JDotN); + } const int ioff = fluxFunction.num_equations * dof1; @@ -360,36 +384,325 @@ void HyperbolicFormIntegrator::AssembleFaceGrad( } } - // Trial side 2 + if (dof2) + { + // Trial side 2 + + // Compute hat(J) using evaluated quantities + numFlux.Grad(2, state1, state2, nor, Tr, JDotN); + + const int joff = ioff; + + for (int di = 0; di < fluxFunction.num_equations; di++) + for (int dj = 0; dj < fluxFunction.num_equations; dj++) + { + // pre-multiply integration weight to Jacobian + const real_t w = +ip.weight * sign * JDotN(di,dj); + for (int j = 0; j < dof2; j++) + { + // Test side 1 + for (int i = 0; i < dof1; i++) + { + elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j); + } + + // Test side 2 + for (int i = 0; i < dof2; i++) + { + elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j); + } + } + } + } + } +} + +BdrHyperbolicDirichletIntegrator::BdrHyperbolicDirichletIntegrator( + const NumericalFlux &numFlux, + VectorCoefficient &bdrState, + const int IntOrderOffset, + real_t sign) + : NonlinearFormIntegrator(), + numFlux(numFlux), + fluxFunction(numFlux.GetFluxFunction()), + u_vcoeff(bdrState), + IntOrderOffset(IntOrderOffset), + sign(sign), + num_equations(fluxFunction.num_equations) +{ + MFEM_VERIFY(fluxFunction.num_equations == bdrState.GetVDim(), + "Flux function does not match the vector dimension of the coefficient!"); +#ifndef MFEM_THREAD_SAFE + state_in.SetSize(num_equations); + state_out.SetSize(num_equations); + fluxN.SetSize(num_equations); + JDotN.SetSize(num_equations); + nor.SetSize(fluxFunction.dim); +#endif + ResetMaxCharSpeed(); +} + +void BdrHyperbolicDirichletIntegrator::AssembleFaceVector( + const FiniteElement &el, const FiniteElement &, + FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect) +{ + MFEM_ASSERT(Tr.Elem2No < 0, "Not a boundary face!"); + + // current elements' the number of degrees of freedom + // does not consider the number of equations + const int dof = el.GetDof(); + +#ifdef MFEM_THREAD_SAFE + // Local storage for element integration + + // shape function value at an integration point + Vector shape(dof); + // normal vector (usually not a unit vector) + Vector nor(Tr.GetSpaceDim()); + // state value at an integration point - interior + Vector state_in(num_equations); + // state value at an integration point - boundary + Vector state_out(num_equations); + // hat(F)(u,x) + Vector fluxN(num_equations); +#else + shape.SetSize(dof); +#endif + + elvect.SetSize(dof * num_equations); + elvect = 0.0; + + const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations); + + DenseMatrix elvect_mat(elvect.GetData(), dof, num_equations); + + // Obtain integration rule. If integration is rule is given, then use it. + // Otherwise, get (2*p + IntOrderOffset) order integration rule + const IntegrationRule *ir = IntRule; + if (!ir) + { + const int order = 2*el.GetOrder() + IntOrderOffset; + ir = &IntRules.Get(Tr.GetGeometryType(), order); + } + // loop over integration points + for (int i = 0; i < ir->GetNPoints(); i++) + { + const IntegrationPoint &ip = ir->IntPoint(i); + + Tr.SetAllIntPoints(&ip); // set face and element int. points + + // Calculate basis functions at the face + el.CalcShape(Tr.GetElement1IntPoint(), shape); + + // Interpolate elfun at the point + elfun_mat.MultTranspose(shape, state_in); + + // Evaluate boundary state at the point + u_vcoeff.Eval(state_out, Tr, ip); + + // Get the normal vector and the flux on the face + if (nor.Size() == 1) // if 1D, use 1 or -1. + { + nor(0) = 2*Tr.GetElement1IntPoint().x - 1.; + } + else + { + CalcOrtho(Tr.Jacobian(), nor); + } + // Compute F(u+, x) and F(u_b, x) with maximum characteristic speed + // Compute hat(F) using evaluated quantities + const real_t speed = numFlux.Eval(state_in, state_out, nor, Tr, fluxN); + + // Update the global max char speed + max_char_speed = std::max(speed, max_char_speed); + + // pre-multiply integration weight to flux + AddMult_a_VWt(-ip.weight*sign, shape, fluxN, elvect_mat); + } +} + +void BdrHyperbolicDirichletIntegrator::AssembleFaceGrad( + const FiniteElement &el, const FiniteElement &, + FaceElementTransformations &Tr, const Vector &elfun, DenseMatrix &elmat) +{ + // current elements' the number of degrees of freedom + // does not consider the number of equations + const int dof = el.GetDof(); + +#ifdef MFEM_THREAD_SAFE + // Local storage for element integration + + // shape function value at an integration point + Vector shape(dof); + // normal vector (usually not a unit vector) + Vector nor(Tr.GetSpaceDim()); + // state value at an integration point - interior + Vector state_in(num_equations); + // state value at an integration point - boundary + Vector state_out(num_equations); + // hat(J)(u,x) + DenseMatrix JDotN(num_equations); +#else + shape.SetSize(dof); +#endif + + elmat.SetSize(dof * num_equations); + elmat = 0.0; + + const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations); + + // Obtain integration rule. If integration is rule is given, then use it. + // Otherwise, get (2*p + IntOrderOffset) order integration rule + const IntegrationRule *ir = IntRule; + if (!ir) + { + const int order = 2*el.GetOrder() + IntOrderOffset; + ir = &IntRules.Get(Tr.GetGeometryType(), order); + } + // loop over integration points + for (int q = 0; q < ir->GetNPoints(); q++) + { + const IntegrationPoint &ip = ir->IntPoint(q); + + Tr.SetAllIntPoints(&ip); // set face and element int. points + + // Calculate basis functions at the face + el.CalcShape(Tr.GetElement1IntPoint(), shape); + + // Interpolate elfun at the point + elfun_mat.MultTranspose(shape, state_in); + + // Evaluate boundary state at the point + u_vcoeff.Eval(state_out, Tr, ip); + + // Get the normal vector and the flux on the face + if (nor.Size() == 1) // if 1D, use 1 or -1. + { + nor(0) = 2*Tr.GetElement1IntPoint().x - 1.; + } + else + { + CalcOrtho(Tr.Jacobian(), nor); + } // Compute hat(J) using evaluated quantities - numFlux.Grad(2, state1, state2, nor, Tr, JDotN); - - const int joff = ioff; + numFlux.Grad(1, state_in, state_out, nor, Tr, JDotN); for (int di = 0; di < fluxFunction.num_equations; di++) for (int dj = 0; dj < fluxFunction.num_equations; dj++) { // pre-multiply integration weight to Jacobian - const real_t w = +ip.weight * sign * JDotN(di,dj); - for (int j = 0; j < dof2; j++) - { - // Test side 1 - for (int i = 0; i < dof1; i++) + const real_t w = -ip.weight * sign * JDotN(di,dj); + for (int j = 0; j < dof; j++) + for (int i = 0; i < dof; i++) { - elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j); + elmat(i+dof*di, j+dof*dj) += w * shape(i) * shape(j); } - - // Test side 2 - for (int i = 0; i < dof2; i++) - { - elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j); - } - } } } } +BoundaryHyperbolicFlowIntegrator::BoundaryHyperbolicFlowIntegrator( + const FluxFunction &flux, VectorCoefficient &u, real_t alpha_, real_t beta_, + const int IntOrderOffset_) + : fluxFunction(flux), u_vcoeff(u), alpha(alpha_), beta(beta_), + IntOrderOffset(IntOrderOffset_) +{ + MFEM_VERIFY(fluxFunction.num_equations == u_vcoeff.GetVDim(), + "Flux function does not match the vector dimension of the coefficient!"); +#ifndef MFEM_THREAD_SAFE + state.SetSize(fluxFunction.num_equations); + nor.SetSize(fluxFunction.dim); + fluxN.SetSize(fluxFunction.num_equations); +#endif + ResetMaxCharSpeed(); +} + +void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect( + const FiniteElement &el, ElementTransformation &Tr, Vector &elvect) +{ + mfem_error("BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect\n" + " is not implemented as boundary integrator!\n" + " Use LinearForm::AddBdrFaceIntegrator instead of\n" + " LinearForm::AddBoundaryIntegrator."); +} + +void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect( + const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect) +{ + // current elements' the number of degrees of freedom + // does not consider the number of equations + const int dof = el.GetDof(); + +#ifdef MFEM_THREAD_SAFE + // Local storage for element integration + + // shape function value at an integration point + Vector shape(dof); + // state value at an integration point + Vector state(fluxFunction.num_equations); + // normal vector (usually not a unit vector) + Vector nor(Tr.GetSpaceDim()); + // hat(F)(u,x) + Vector fluxN(fluxFunction.num_equations); +#else + shape.SetSize(dof); +#endif + + elvect.SetSize(dof * fluxFunction.num_equations); + elvect = 0.0; + + DenseMatrix elvect_mat(elvect.GetData(), dof, fluxFunction.num_equations); + + // Obtain integration rule. If integration is rule is given, then use it. + // Otherwise, get (2*p + IntOrderOffset) order integration rule + const IntegrationRule *ir = IntRule; + if (!ir) + { + const int order = 2*el.GetOrder() + IntOrderOffset; + ir = &IntRules.Get(Tr.GetGeometryType(), order); + } + // loop over integration points + for (int i = 0; i < ir->GetNPoints(); i++) + { + const IntegrationPoint &ip = ir->IntPoint(i); + + Tr.SetAllIntPoints(&ip); // set face and element int. points + + // Calculate basis functions on both elements at the face + el.CalcShape(Tr.GetElement1IntPoint(), shape); + + // Evaluate the coefficient at the point + u_vcoeff.Eval(state, Tr, ip); + + // Get the normal vector and the flux on the face + if (nor.Size() == 1) // if 1D, use 1 or -1. + { + nor(0) = 2*Tr.GetElement1IntPoint().x - 1.; + } + else + { + CalcOrtho(Tr.Jacobian(), nor); + } + // Compute F(u, x) with maximum characteristic speed + const real_t speed = fluxFunction.ComputeFluxDotN(state, nor, Tr, fluxN); + + // Update the global max char speed + max_char_speed = std::max(speed, max_char_speed); + + // pre-multiply integration weight to flux + const real_t a = 0.5 * alpha * ip.weight; + const real_t b = beta * ip.weight; + + for (int n = 0; n < fluxFunction.num_equations; n++) + { + fluxN(n) = a * fluxN(n) - b * fabs(fluxN(n)); + } + + AddMultVWt(shape, fluxN, elvect_mat); + } +} + real_t FluxFunction::ComputeFluxDotN(const Vector &U, const Vector &normal, FaceElementTransformations &Tr, diff --git a/fem/hyperbolic.hpp b/fem/hyperbolic.hpp index 71e679b89f..5b312b60bd 100644 --- a/fem/hyperbolic.hpp +++ b/fem/hyperbolic.hpp @@ -306,12 +306,14 @@ MFEM_DEPRECATED typedef NumericalFlux RiemannSolver; class HyperbolicFormIntegrator : public NonlinearFormIntegrator { private: - // The maximum characteristic speed, updated during element/face vector assembly - real_t max_char_speed; const NumericalFlux &numFlux; // Numerical flux that maps F(u±,x) to F̂ const FluxFunction &fluxFunction; const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset. const real_t sign; + + // The maximum characteristic speed, updated during element/face vector assembly + real_t max_char_speed; + #ifndef MFEM_THREAD_SAFE // Local storage for element integration Vector shape; // shape function value at an integration point @@ -331,8 +333,9 @@ private: public: const int num_equations; // the number of equations + /** - * @brief Construct a new Hyperbolic Form Integrator object + * @brief Construct a new HyperbolicFormIntegrator object * * @param[in] numFlux numerical flux * @param[in] IntOrderOffset integration order offset @@ -343,21 +346,14 @@ public: const int IntOrderOffset = 0, const real_t sign = 1.); - /** - * @brief Reset the Max Char Speed 0 - * - */ - void ResetMaxCharSpeed() - { - max_char_speed = 0.0; - } + /// Reset the maximum characteristic speed to zero + void ResetMaxCharSpeed() { max_char_speed = 0.0; } - real_t GetMaxCharSpeed() - { - return max_char_speed; - } + /// Get the maximum characteristic speed + real_t GetMaxCharSpeed() const { return max_char_speed; } - const FluxFunction &GetFluxFunction() { return fluxFunction; } + /// Get the associated flux function + const FluxFunction &GetFluxFunction() const { return fluxFunction; } /** * @brief Implements (F(u), ∇v) with abstract F computed by @@ -416,6 +412,182 @@ public: const Vector &elfun, DenseMatrix &elmat) override; }; +/** + * @brief Abstract boundary hyperbolic form integrator, assembling + * term for scalar finite elements at the boundary. + * + * This form integrator is coupled with a NumericalFlux that implements the + * numerical flux F̂ at the boundary faces. The flux F is obtained from the + * FluxFunction assigned to the aforementioned NumericalFlux with the given + * boundary coefficient for the state u_b. + * + * Note the class can be used for imposing conditions on interior interfaces. + */ +class BdrHyperbolicDirichletIntegrator : public NonlinearFormIntegrator +{ +private: + const NumericalFlux &numFlux; // Numerical flux that maps F to F̂ + const FluxFunction &fluxFunction; + VectorCoefficient &u_vcoeff; // Boundary state vector coefficient + const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset. + const real_t sign; + + // The maximum characteristic speed, updated during element/face vector assembly + real_t max_char_speed; + +#ifndef MFEM_THREAD_SAFE + // Local storage for element integration + Vector shape; // shape function value at an integration point + Vector state_in; // state value at an integration point - interior + Vector state_out; // state value at an integration point - boundary + Vector nor; // normal vector, see mfem::CalcOrtho() + Vector fluxN; // F̂(u⁻,u_b,x) n + DenseMatrix JDotN; // Ĵ(u⁻,u_b,x) n +#endif + +public: + const int num_equations; // the number of equations + + /** + * @brief Construct a new BdrHyperbolicDirichletIntegrator object + * + * @param[in] numFlux numerical flux + * @param[in] bdrState boundary state coefficient + * @param[in] IntOrderOffset integration order offset + * @param[in] sign sign of the convection term + */ + BdrHyperbolicDirichletIntegrator( + const NumericalFlux &numFlux, + VectorCoefficient &bdrState, + const int IntOrderOffset = 0, + const real_t sign = 1.); + + /// Reset the maximum characteristic speed to zero + void ResetMaxCharSpeed() { max_char_speed = 0.0; } + + /// Get the maximum characteristic speed + real_t GetMaxCharSpeed() const { return max_char_speed; } + + /// Get the associated flux function + const FluxFunction &GetFluxFunction() const { return fluxFunction; } + + /** + * @brief Implements <-F̂(u⁻,u_b,x) n, [v]> with abstract F̂ computed by + * NumericalFlux::Eval() of the numerical flux object + * + * @param[in] el1 finite element of the interior element + * @param[in] el2 not used + * @param[in] Tr face element transformations + * @param[in] elfun local coefficient of basis for the interior element + * @param[out] elvect evaluated dual vector <-F̂(u⁻,u_b,x) n, [v]> + */ + void AssembleFaceVector(const FiniteElement &el1, + const FiniteElement &el2, + FaceElementTransformations &Tr, + const Vector &elfun, Vector &elvect) override; + + /** + * @brief Implements <-Ĵ(u⁻,u_b,x) n, [v]> with abstract Ĵ computed by + * NumericalFlux::Grad() of the numerical flux object + * + * @param[in] el1 finite element of the interior element + * @param[in] el2 not used + * @param[in] Tr face element transformations + * @param[in] elfun local coefficient of basis for the interior element + * @param[out] elmat evaluated Jacobian matrix <-Ĵ(u⁻,u_b,x) n, [v]> + */ + void AssembleFaceGrad(const FiniteElement &el1, + const FiniteElement &el2, + FaceElementTransformations &Tr, + const Vector &elfun, DenseMatrix &elmat) override; +}; + +/** + * @brief Abstract boundary hyperbolic linear form integrator, assembling + * <ɑ/2 F(u,x) n - β |F(u,x) n|, v> terms for scalar finite elements. + * + * This form integrator is coupled with a FluxFunction that evaluates the + * flux F at the boundary. + * + * Note the upwinding is performed component-wise. For general boundary + * integration with a numerical flux, see BdrHyperbolicDirichletIntegrator. + */ +class BoundaryHyperbolicFlowIntegrator : public LinearFormIntegrator +{ + const FluxFunction &fluxFunction; + VectorCoefficient &u_vcoeff; + const real_t alpha, beta; + const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset. + + // The maximum characteristic speed, updated during face vector assembly + real_t max_char_speed; + +#ifndef MFEM_THREAD_SAFE + // Local storage for element integration + Vector shape; // shape function value at an integration point + Vector state; // state value at an integration point + Vector nor; // normal vector, see mfem::CalcOrtho() + Vector fluxN; // F(u,x) n +#endif + +public: + /** + * @brief Construct a new BoundaryHyperbolicFlowIntegrator object + * + * @param[in] flux flux function + * @param[in] u vector state coefficient + * @param[in] alpha ɑ coefficient (β = ɑ/2) + * @param[in] IntOrderOffset integration order offset + */ + BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u, + real_t alpha = -1., int IntOrderOffset = 0) + : BoundaryHyperbolicFlowIntegrator(flux, u, alpha, alpha/2., IntOrderOffset) { } + + /** + * @brief Construct a new BoundaryHyperbolicFlowIntegrator object + * + * @param[in] flux flux function + * @param[in] u vector state coefficient + * @param[in] alpha ɑ coefficient + * @param[in] beta β coefficient + * @param[in] IntOrderOffset integration order offset + */ + BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u, + real_t alpha, real_t beta, int IntOrderOffset = 0); + + /// Reset the maximum characteristic speed to zero + void ResetMaxCharSpeed() { max_char_speed = 0.0; } + + /// Get the maximum characteristic speed + real_t GetMaxCharSpeed() const { return max_char_speed; } + + /// Get the associated flux function + const FluxFunction &GetFluxFunction() const { return fluxFunction; } + + using LinearFormIntegrator::AssembleRHSElementVect; + + /** + * @warning Boundary element integration not implemented, use + * AssembleRHSElementVect(const FiniteElement&, + * FaceElementTransformations &, Vector &) instead + */ + void AssembleRHSElementVect(const FiniteElement &el, + ElementTransformation &Tr, + Vector &elvect) override; + + /** + * @brief Implements <-F(u,x) n, v> with abstract F computed by + * FluxFunction::ComputeFluxDotN() of the flux function object + * + * @param[in] el finite element + * @param[in] Tr face element transformations + * @param[out] elvect evaluated dual vector + */ + void AssembleRHSElementVect(const FiniteElement &el, + FaceElementTransformations &Tr, + Vector &elvect) override; +}; + /** * @brief Rusanov flux, also known as local Lax-Friedrichs, diff --git a/fem/integ/bilininteg_convection_kernels.hpp b/fem/integ/bilininteg_convection_kernels.hpp new file mode 100644 index 0000000000..6530584f7d --- /dev/null +++ b/fem/integ/bilininteg_convection_kernels.hpp @@ -0,0 +1,1308 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_BILININTEG_CONVECTION_KERNELS_HPP +#define MFEM_BILININTEG_CONVECTION_KERNELS_HPP + +#include "../../general/forall.hpp" +#include "../bilininteg.hpp" +#include "../gridfunc.hpp" +#include "../qfunction.hpp" +#include "../ceed/integrators/convection/convection.hpp" + +/// \cond DO_NOT_DOCUMENT +namespace mfem +{ + +// PA Convection Apply 2D kernel +template static +void PAConvectionApply2D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto G = Reshape(g.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); + auto x = Reshape(x_.Read(), D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + + real_t u[max_D1D][max_D1D]; + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + u[dy][dx] = x(dx,dy,e); + } + } + real_t Bu[max_D1D][max_Q1D]; + real_t Gu[max_D1D][max_Q1D]; + for (int dy = 0; dy < D1D; ++dy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + Bu[dy][qx] = 0.0; + Gu[dy][qx] = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t gx = G(qx,dx); + const real_t x = u[dy][dx]; + Bu[dy][qx] += bx * x; + Gu[dy][qx] += gx * x; + } + } + } + real_t GBu[max_Q1D][max_Q1D]; + real_t BGu[max_Q1D][max_Q1D]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + GBu[qy][qx] = 0.0; + BGu[qy][qx] = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + const real_t gx = G(qy,dy); + GBu[qy][qx] += gx * Bu[dy][qx]; + BGu[qy][qx] += bx * Gu[dy][qx]; + } + } + } + // Calculate Dxy, xDy in plane + real_t DGu[max_Q1D][max_Q1D]; + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t O1 = op(qx,qy,0,e); + const real_t O2 = op(qx,qy,1,e); + + const real_t gradX = BGu[qy][qx]; + const real_t gradY = GBu[qy][qx]; + + DGu[qy][qx] = (O1 * gradX) + (O2 * gradY); + } + } + real_t BDGu[max_D1D][max_Q1D]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int dy = 0; dy < D1D; ++dy) + { + BDGu[dy][qx] = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t w = Bt(dy,qy); + BDGu[dy][qx] += w * DGu[qy][qx]; + } + } + } + for (int dx = 0; dx < D1D; ++dx) + { + for (int dy = 0; dy < D1D; ++dy) + { + real_t BBDGu = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t w = Bt(dx,qx); + BBDGu += w * BDGu[dy][qx]; + } + y(dx,dy,e) += BBDGu; + } + } + }); +} + +// Optimized PA Convection Apply 2D kernel +template static +void SmemPAConvectionApply2D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto G = Reshape(g.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); + auto x = Reshape(x_.Read(), D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); + mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e) + { + const int tidz = MFEM_THREAD_ID(z); + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + // constexpr int MDQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D; + MFEM_SHARED real_t u[NBZ][max_D1D][max_D1D]; + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(dx,x,D1D) + { + // e is really equal to e+tidz + u[tidz][dy][dx] = x(dx,dy,e); + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t Bu[NBZ][max_D1D][max_Q1D]; + MFEM_SHARED real_t Gu[NBZ][max_D1D][max_Q1D]; + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + Bu[tidz][dy][qx] = 0.0; + Gu[tidz][dy][qx] = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t gx = G(qx,dx); + const real_t x = u[tidz][dy][dx]; + Bu[tidz][dy][qx] += bx * x; + Gu[tidz][dy][qx] += gx * x; + } + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t GBu[NBZ][max_Q1D][max_Q1D]; + MFEM_SHARED real_t BGu[NBZ][max_Q1D][max_Q1D]; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + GBu[tidz][qy][qx] = 0.0; + BGu[tidz][qy][qx] = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + const real_t gx = G(qy,dy); + GBu[tidz][qy][qx] += gx * Bu[tidz][dy][qx]; + BGu[tidz][qy][qx] += bx * Gu[tidz][dy][qx]; + } + } + } + MFEM_SYNC_THREAD; + // Calculate Dxy, xDy in plane + MFEM_SHARED real_t DGu[NBZ][max_Q1D][max_Q1D]; + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const real_t O1 = op(qx,qy,0,e); + const real_t O2 = op(qx,qy,1,e); + + const real_t gradX = BGu[tidz][qy][qx]; + const real_t gradY = GBu[tidz][qy][qx]; + + DGu[tidz][qy][qx] = (O1 * gradX) + (O2 * gradY); + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t BDGu[NBZ][max_D1D][max_Q1D]; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + BDGu[tidz][dy][qx] = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t w = Bt(dy,qy); + BDGu[tidz][dy][qx] += w * DGu[tidz][qy][qx]; + } + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(dx,x,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + real_t BBDGu = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t w = Bt(dx,qx); + BBDGu += w * BDGu[tidz][dy][qx]; + } + y(dx,dy,e) += BBDGu; + } + } + }); +} + +// PA Convection Apply 3D kernel +template static +void PAConvectionApply3D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto G = Reshape(g.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); + auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + + real_t u[max_D1D][max_D1D][max_D1D]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + u[dz][dy][dx] = x(dx,dy,dz,e); + } + } + } + real_t Bu[max_D1D][max_D1D][max_Q1D]; + real_t Gu[max_D1D][max_D1D][max_Q1D]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int dy = 0; dy < D1D; ++dy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + Bu[dz][dy][qx] = 0.0; + Gu[dz][dy][qx] = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t gx = G(qx,dx); + const real_t x = u[dz][dy][dx]; + Bu[dz][dy][qx] += bx * x; + Gu[dz][dy][qx] += gx * x; + } + } + } + } + real_t BBu[max_D1D][max_Q1D][max_Q1D]; + real_t GBu[max_D1D][max_Q1D][max_Q1D]; + real_t BGu[max_D1D][max_Q1D][max_Q1D]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + BBu[dz][qy][qx] = 0.0; + GBu[dz][qy][qx] = 0.0; + BGu[dz][qy][qx] = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + const real_t gx = G(qy,dy); + BBu[dz][qy][qx] += bx * Bu[dz][dy][qx]; + GBu[dz][qy][qx] += gx * Bu[dz][dy][qx]; + BGu[dz][qy][qx] += bx * Gu[dz][dy][qx]; + } + } + } + } + real_t GBBu[max_Q1D][max_Q1D][max_Q1D]; + real_t BGBu[max_Q1D][max_Q1D][max_Q1D]; + real_t BBGu[max_Q1D][max_Q1D][max_Q1D]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qz = 0; qz < Q1D; ++qz) + { + GBBu[qz][qy][qx] = 0.0; + BGBu[qz][qy][qx] = 0.0; + BBGu[qz][qy][qx] = 0.0; + for (int dz = 0; dz < D1D; ++dz) + { + const real_t bx = B(qz,dz); + const real_t gx = G(qz,dz); + GBBu[qz][qy][qx] += gx * BBu[dz][qy][qx]; + BGBu[qz][qy][qx] += bx * GBu[dz][qy][qx]; + BBGu[qz][qy][qx] += bx * BGu[dz][qy][qx]; + } + } + } + } + // Calculate Dxy, xDy in plane + real_t DGu[max_Q1D][max_Q1D][max_Q1D]; + for (int qz = 0; qz < Q1D; ++qz) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t O1 = op(qx,qy,qz,0,e); + const real_t O2 = op(qx,qy,qz,1,e); + const real_t O3 = op(qx,qy,qz,2,e); + + const real_t gradX = BBGu[qz][qy][qx]; + const real_t gradY = BGBu[qz][qy][qx]; + const real_t gradZ = GBBu[qz][qy][qx]; + + DGu[qz][qy][qx] = (O1 * gradX) + (O2 * gradY) + (O3 * gradZ); + } + } + } + real_t BDGu[max_D1D][max_Q1D][max_Q1D]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int dz = 0; dz < D1D; ++dz) + { + BDGu[dz][qy][qx] = 0.0; + for (int qz = 0; qz < Q1D; ++qz) + { + const real_t w = Bt(dz,qz); + BDGu[dz][qy][qx] += w * DGu[qz][qy][qx]; + } + } + } + } + real_t BBDGu[max_D1D][max_D1D][max_Q1D]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int qx = 0; qx < Q1D; ++qx) + { + for (int dy = 0; dy < D1D; ++dy) + { + BBDGu[dz][dy][qx] = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t w = Bt(dy,qy); + BBDGu[dz][dy][qx] += w * BDGu[dz][qy][qx]; + } + } + } + } + for (int dz = 0; dz < D1D; ++dz) + { + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + real_t BBBDGu = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t w = Bt(dx,qx); + BBBDGu += w * BBDGu[dz][dy][qx]; + } + y(dx,dy,dz,e) += BBBDGu; + } + } + } + }); +} + +// Optimized PA Convection Apply 3D kernel +template static +void SmemPAConvectionApply3D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto G = Reshape(g.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); + auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); + mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int max_DQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D; + MFEM_SHARED real_t sm0[max_DQ*max_DQ*max_DQ]; + MFEM_SHARED real_t sm1[max_DQ*max_DQ*max_DQ]; + MFEM_SHARED real_t sm2[max_DQ*max_DQ*max_DQ]; + MFEM_SHARED real_t sm3[max_DQ*max_DQ*max_DQ]; + MFEM_SHARED real_t sm4[max_DQ*max_DQ*max_DQ]; + MFEM_SHARED real_t sm5[max_DQ*max_DQ*max_DQ]; + + real_t (*u)[max_D1D][max_D1D] = (real_t (*)[max_D1D][max_D1D]) sm0; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(dx,x,D1D) + { + u[dz][dy][dx] = x(dx,dy,dz,e); + } + } + } + MFEM_SYNC_THREAD; + real_t (*Bu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm1; + real_t (*Gu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm2; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + real_t Bu_ = 0.0; + real_t Gu_ = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t gx = G(qx,dx); + const real_t x = u[dz][dy][dx]; + Bu_ += bx * x; + Gu_ += gx * x; + } + Bu[dz][dy][qx] = Bu_; + Gu[dz][dy][qx] = Gu_; + } + } + } + MFEM_SYNC_THREAD; + real_t (*BBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm3; + real_t (*GBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm4; + real_t (*BGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm5; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + real_t BBu_ = 0.0; + real_t GBu_ = 0.0; + real_t BGu_ = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + const real_t gx = G(qy,dy); + BBu_ += bx * Bu[dz][dy][qx]; + GBu_ += gx * Bu[dz][dy][qx]; + BGu_ += bx * Gu[dz][dy][qx]; + } + BBu[dz][qy][qx] = BBu_; + GBu[dz][qy][qx] = GBu_; + BGu[dz][qy][qx] = BGu_; + } + } + } + MFEM_SYNC_THREAD; + real_t (*GBBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm0; + real_t (*BGBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm1; + real_t (*BBGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm2; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qz,z,Q1D) + { + real_t GBBu_ = 0.0; + real_t BGBu_ = 0.0; + real_t BBGu_ = 0.0; + for (int dz = 0; dz < D1D; ++dz) + { + const real_t bx = B(qz,dz); + const real_t gx = G(qz,dz); + GBBu_ += gx * BBu[dz][qy][qx]; + BGBu_ += bx * GBu[dz][qy][qx]; + BBGu_ += bx * BGu[dz][qy][qx]; + } + GBBu[qz][qy][qx] = GBBu_; + BGBu[qz][qy][qx] = BGBu_; + BBGu[qz][qy][qx] = BBGu_; + } + } + } + MFEM_SYNC_THREAD; + real_t (*DGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm3; + MFEM_FOREACH_THREAD(qz,z,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const real_t O1 = op(qx,qy,qz,0,e); + const real_t O2 = op(qx,qy,qz,1,e); + const real_t O3 = op(qx,qy,qz,2,e); + + const real_t gradX = BBGu[qz][qy][qx]; + const real_t gradY = BGBu[qz][qy][qx]; + const real_t gradZ = GBBu[qz][qy][qx]; + + DGu[qz][qy][qx] = (O1 * gradX) + (O2 * gradY) + (O3 * gradZ); + } + } + } + MFEM_SYNC_THREAD; + real_t (*BDGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm4; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(dz,z,D1D) + { + real_t BDGu_ = 0.0; + for (int qz = 0; qz < Q1D; ++qz) + { + const real_t w = Bt(dz,qz); + BDGu_ += w * DGu[qz][qy][qx]; + } + BDGu[dz][qy][qx] = BDGu_; + } + } + } + MFEM_SYNC_THREAD; + real_t (*BBDGu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm5; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + real_t BBDGu_ = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t w = Bt(dy,qy); + BBDGu_ += w * BDGu[dz][qy][qx]; + } + BBDGu[dz][dy][qx] = BBDGu_; + } + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(dx,x,D1D) + { + real_t BBBDGu = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t w = Bt(dx,qx); + BBBDGu += w * BBDGu[dz][dy][qx]; + } + y(dx,dy,dz,e) += BBBDGu; + } + } + } + }); +} + +// PA Convection Apply 2D kernel +template static +void PAConvectionApplyT2D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto Gt = Reshape(gt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); + auto x = Reshape(x_.Read(), D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + + real_t u[max_D1D][max_D1D]; + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + u[dy][dx] = x(dx,dy,e); + } + } + real_t Bu[max_D1D][max_Q1D]; + for (int dy = 0; dy < D1D; ++dy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + Bu[dy][qx] = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t x = u[dy][dx]; + Bu[dy][qx] += bx * x; + } + } + } + real_t BBu[max_Q1D][max_Q1D]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + BBu[qy][qx] = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + BBu[qy][qx] += bx * Bu[dy][qx]; + } + } + } + // Calculate Dxy, xDy in plane + real_t DBu[max_Q1D][max_Q1D][2]; + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t O1 = op(qx,qy,0,e); + const real_t O2 = op(qx,qy,1,e); + + const real_t X = BBu[qy][qx]; + + DBu[qy][qx][0] = O1 * X; + DBu[qy][qx][1] = O2 * X; + } + } + real_t GDBu[max_D1D][max_Q1D][2]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int dy = 0; dy < D1D; ++dy) + { + GDBu[dy][qx][0] = 0.0; + GDBu[dy][qx][1] = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t by = Bt(dy,qy); + const real_t gy = Gt(dy,qy); + GDBu[dy][qx][0] += by * DBu[qy][qx][0]; + GDBu[dy][qx][1] += gy * DBu[qy][qx][1]; + } + } + } + for (int dx = 0; dx < D1D; ++dx) + { + for (int dy = 0; dy < D1D; ++dy) + { + real_t res = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t bx = Bt(dx,qx); + const real_t gx = Gt(dx,qx); + res += gx * GDBu[dy][qx][0] + bx * GDBu[dy][qx][1]; + } + y(dx,dy,e) += res; + } + } + }); +} + +// Optimized PA Convection Apply 2D kernel +template static +void SmemPAConvectionApplyT2D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto Gt = Reshape(gt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); + auto x = Reshape(x_.Read(), D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); + mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e) + { + const int tidz = MFEM_THREAD_ID(z); + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + MFEM_SHARED real_t u[NBZ][max_D1D][max_D1D]; + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(dx,x,D1D) + { + // e is really equal to e+tidz + u[tidz][dy][dx] = x(dx,dy,e); + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t Bu[NBZ][max_D1D][max_Q1D]; + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + Bu[tidz][dy][qx] = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t x = u[tidz][dy][dx]; + Bu[tidz][dy][qx] += bx * x; + } + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t BBu[NBZ][max_Q1D][max_Q1D]; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + BBu[tidz][qy][qx] = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + BBu[tidz][qy][qx] += bx * Bu[tidz][dy][qx]; + } + } + } + MFEM_SYNC_THREAD; + // Calculate Dxy, xDy in plane + MFEM_SHARED real_t DBu[NBZ][max_Q1D][max_Q1D][2]; + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const real_t O1 = op(qx,qy,0,e); + const real_t O2 = op(qx,qy,1,e); + + const real_t X = BBu[tidz][qy][qx]; + + DBu[tidz][qy][qx][0] = O1 * X; + DBu[tidz][qy][qx][1] = O2 * X; + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t GDBu[NBZ][max_D1D][max_Q1D][2]; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + GDBu[tidz][dy][qx][0] = 0.0; + GDBu[tidz][dy][qx][1] = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t by = Bt(dy,qy); + const real_t gy = Gt(dy,qy); + GDBu[tidz][dy][qx][0] += by * DBu[tidz][qy][qx][0]; + GDBu[tidz][dy][qx][1] += gy * DBu[tidz][qy][qx][1]; + } + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(dx,x,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + real_t res = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t bx = Bt(dx,qx); + const real_t gx = Gt(dx,qx); + res += gx * GDBu[tidz][dy][qx][0] + bx * GDBu[tidz][dy][qx][1]; + } + y(dx,dy,e) += res; + } + } + }); +} + +// PA Convection Apply 3D kernel +template static +void PAConvectionApplyT3D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto Gt = Reshape(gt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); + auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + + real_t u[max_D1D][max_D1D][max_D1D]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + u[dz][dy][dx] = x(dx,dy,dz,e); + } + } + } + real_t Bu[max_D1D][max_D1D][max_Q1D]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int dy = 0; dy < D1D; ++dy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + Bu[dz][dy][qx] = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t x = u[dz][dy][dx]; + Bu[dz][dy][qx] += bx * x; + } + } + } + } + real_t BBu[max_D1D][max_Q1D][max_Q1D]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + BBu[dz][qy][qx] = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + BBu[dz][qy][qx] += bx * Bu[dz][dy][qx]; + } + } + } + } + real_t BBBu[max_Q1D][max_Q1D][max_Q1D]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qz = 0; qz < Q1D; ++qz) + { + BBBu[qz][qy][qx] = 0.0; + for (int dz = 0; dz < D1D; ++dz) + { + const real_t bx = B(qz,dz); + BBBu[qz][qy][qx] += bx * BBu[dz][qy][qx]; + } + } + } + } + // Calculate Dxy, xDy in plane + real_t DBu[max_Q1D][max_Q1D][max_Q1D][3]; + for (int qz = 0; qz < Q1D; ++qz) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t O1 = op(qx,qy,qz,0,e); + const real_t O2 = op(qx,qy,qz,1,e); + const real_t O3 = op(qx,qy,qz,2,e); + + const real_t X = BBBu[qz][qy][qx]; + + DBu[qz][qy][qx][0] = O1 * X; + DBu[qz][qy][qx][1] = O2 * X; + DBu[qz][qy][qx][2] = O3 * X; + } + } + } + real_t GDBu[max_D1D][max_Q1D][max_Q1D][3]; + for (int qx = 0; qx < Q1D; ++qx) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int dz = 0; dz < D1D; ++dz) + { + GDBu[dz][qy][qx][0] = 0.0; + GDBu[dz][qy][qx][1] = 0.0; + GDBu[dz][qy][qx][2] = 0.0; + for (int qz = 0; qz < Q1D; ++qz) + { + const real_t bz = Bt(dz,qz); + const real_t gz = Gt(dz,qz); + GDBu[dz][qy][qx][0] += bz * DBu[qz][qy][qx][0]; + GDBu[dz][qy][qx][1] += bz * DBu[qz][qy][qx][1]; + GDBu[dz][qy][qx][2] += gz * DBu[qz][qy][qx][2]; + } + } + } + } + real_t GGDBu[max_D1D][max_D1D][max_Q1D][3]; + for (int dz = 0; dz < D1D; ++dz) + { + for (int qx = 0; qx < Q1D; ++qx) + { + for (int dy = 0; dy < D1D; ++dy) + { + GGDBu[dz][dy][qx][0] = 0.0; + GGDBu[dz][dy][qx][1] = 0.0; + GGDBu[dz][dy][qx][2] = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t by = Bt(dy,qy); + const real_t gy = Gt(dy,qy); + GGDBu[dz][dy][qx][0] += by * GDBu[dz][qy][qx][0]; + GGDBu[dz][dy][qx][1] += gy * GDBu[dz][qy][qx][1]; + GGDBu[dz][dy][qx][2] += by * GDBu[dz][qy][qx][2]; + } + } + } + } + for (int dz = 0; dz < D1D; ++dz) + { + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + real_t res = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t bx = Bt(dx,qx); + const real_t gx = Gt(dx,qx); + res += gx * GGDBu[dz][dy][qx][0]; + res += bx * GGDBu[dz][dy][qx][1]; + res += bx * GGDBu[dz][dy][qx][2]; + } + y(dx,dy,dz,e) += res; + } + } + } + }); +} + +// Optimized PA Convection Apply 3D kernel +template static +void SmemPAConvectionApplyT3D(const int ne, + const Array &b, + const Array &g, + const Array &bt, + const Array >, + const Vector &op_, + const Vector &x_, + Vector &y_, + const int d1d = 0, + const int q1d = 0) +{ + const int NE = ne; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto Gt = Reshape(gt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); + auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); + mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int max_DQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D; + MFEM_SHARED real_t sm0[3*max_DQ*max_DQ*max_DQ]; + MFEM_SHARED real_t sm1[3*max_DQ*max_DQ*max_DQ]; + + real_t (*u)[max_D1D][max_D1D] = (real_t (*)[max_D1D][max_D1D]) sm0; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(dx,x,D1D) + { + u[dz][dy][dx] = x(dx,dy,dz,e); + } + } + } + MFEM_SYNC_THREAD; + real_t (*Bu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm1; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + real_t Bu_ = 0.0; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t bx = B(qx,dx); + const real_t x = u[dz][dy][dx]; + Bu_ += bx * x; + } + Bu[dz][dy][qx] = Bu_; + } + } + } + MFEM_SYNC_THREAD; + real_t (*BBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm0; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + real_t BBu_ = 0.0; + for (int dy = 0; dy < D1D; ++dy) + { + const real_t bx = B(qy,dy); + BBu_ += bx * Bu[dz][dy][qx]; + } + BBu[dz][qy][qx] = BBu_; + } + } + } + MFEM_SYNC_THREAD; + real_t (*BBBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm1; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qz,z,Q1D) + { + real_t BBBu_ = 0.0; + for (int dz = 0; dz < D1D; ++dz) + { + const real_t bx = B(qz,dz); + BBBu_ += bx * BBu[dz][qy][qx]; + } + BBBu[qz][qy][qx] = BBBu_; + } + } + } + MFEM_SYNC_THREAD; + real_t (*DBu)[max_Q1D][max_Q1D][3] = (real_t (*)[max_Q1D][max_Q1D][3])sm0; + MFEM_FOREACH_THREAD(qz,z,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + const real_t O1 = op(qx,qy,qz,0,e); + const real_t O2 = op(qx,qy,qz,1,e); + const real_t O3 = op(qx,qy,qz,2,e); + + const real_t X = BBBu[qz][qy][qx]; + + DBu[qz][qy][qx][0] = O1 * X; + DBu[qz][qy][qx][1] = O2 * X; + DBu[qz][qy][qx][2] = O3 * X; + } + } + } + MFEM_SYNC_THREAD; + real_t (*GDBu)[max_Q1D][max_Q1D][3] = (real_t (*)[max_Q1D][max_Q1D][3])sm1; + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(qy,y,Q1D) + { + MFEM_FOREACH_THREAD(dz,z,D1D) + { + real_t GDBu0 = 0.0; + real_t GDBu1 = 0.0; + real_t GDBu2 = 0.0; + for (int qz = 0; qz < Q1D; ++qz) + { + const real_t bz = Bt(dz,qz); + const real_t gz = Gt(dz,qz); + GDBu0 += bz * DBu[qz][qy][qx][0]; + GDBu1 += bz * DBu[qz][qy][qx][1]; + GDBu2 += gz * DBu[qz][qy][qx][2]; + } + GDBu[dz][qy][qx][0] = GDBu0; + GDBu[dz][qy][qx][1] = GDBu1; + GDBu[dz][qy][qx][2] = GDBu2; + } + } + } + MFEM_SYNC_THREAD; + real_t (*GGDBu)[max_D1D][max_Q1D][3] = (real_t (*)[max_D1D][max_Q1D][3])sm0; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(qx,x,Q1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + real_t GGDBu0 = 0.0; + real_t GGDBu1 = 0.0; + real_t GGDBu2 = 0.0; + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t by = Bt(dy,qy); + const real_t gy = Gt(dy,qy); + GGDBu0 += by * GDBu[dz][qy][qx][0]; + GGDBu1 += gy * GDBu[dz][qy][qx][1]; + GGDBu2 += by * GDBu[dz][qy][qx][2]; + } + GGDBu[dz][dy][qx][0] = GGDBu0; + GGDBu[dz][dy][qx][1] = GGDBu1; + GGDBu[dz][dy][qx][2] = GGDBu2; + } + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(dz,z,D1D) + { + MFEM_FOREACH_THREAD(dy,y,D1D) + { + MFEM_FOREACH_THREAD(dx,x,D1D) + { + real_t res = 0.0; + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t bx = Bt(dx,qx); + const real_t gx = Gt(dx,qx); + res += gx * GGDBu[dz][dy][qx][0]; + res += bx * GGDBu[dz][dy][qx][1]; + res += bx * GGDBu[dz][dy][qx][2]; + } + y(dx,dy,dz,e) += res; + } + } + } + }); +} + +namespace convection +{ +constexpr int ipow(int x, int p) { return p == 0 ? 1 : x*ipow(x, p-1); } +constexpr int D(int D1D) { return (11 - D1D) / 2; } +constexpr int NBZ(int D1D) +{ + return ipow(2, D(D1D) >= 0 ? D(D1D) : 0); +} +} + +template +ConvectionIntegrator::ApplyKernelType +ConvectionIntegrator::ApplyPAKernels::Kernel() +{ + if constexpr (DIM == 2) + { + constexpr int T_NBZ = convection::NBZ(T_D1D); + return SmemPAConvectionApply2D; + } + else if constexpr (DIM == 3) + { + return SmemPAConvectionApply3D; + } + MFEM_ABORT(""); +} + +template +ConvectionIntegrator::ApplyKernelType +ConvectionIntegrator::ApplyPATKernels::Kernel() +{ + if constexpr (DIM == 2) + { + constexpr int T_NBZ = convection::NBZ(T_D1D); + return SmemPAConvectionApplyT2D; + } + else if constexpr (DIM == 3) + { + return SmemPAConvectionApplyT3D; + } + MFEM_ABORT(""); +} +} // namespace mfem +/// \endcond DO_NOT_DOCUMENT +#endif diff --git a/fem/integ/bilininteg_convection_pa.cpp b/fem/integ/bilininteg_convection_pa.cpp index fca2897d54..23cceca393 100644 --- a/fem/integ/bilininteg_convection_pa.cpp +++ b/fem/integ/bilininteg_convection_pa.cpp @@ -15,9 +15,48 @@ #include "../qfunction.hpp" #include "../ceed/integrators/convection/convection.hpp" +#include "bilininteg_convection_kernels.hpp" + +/// \cond DO_NOT_DOCUMENT namespace mfem { +ConvectionIntegrator::ConvectionIntegrator(VectorCoefficient &q, real_t a) + : Q(&q), alpha(a) +{ + static Kernels kernels; +} + +ConvectionIntegrator::Kernels::Kernels() +{ + // 2D + // Q = P + 1 + ConvectionIntegrator::AddSpecialization<2, 2, 2>(); + ConvectionIntegrator::AddSpecialization<2, 3, 3>(); + ConvectionIntegrator::AddSpecialization<2, 4, 4>(); + ConvectionIntegrator::AddSpecialization<2, 5, 5>(); + ConvectionIntegrator::AddSpecialization<2, 6, 6>(); + // Q = P + 2 + ConvectionIntegrator::AddSpecialization<2, 2, 3>(); + ConvectionIntegrator::AddSpecialization<2, 3, 4>(); + ConvectionIntegrator::AddSpecialization<2, 4, 5>(); + ConvectionIntegrator::AddSpecialization<2, 5, 6>(); + ConvectionIntegrator::AddSpecialization<2, 6, 7>(); + // 3D + // Q = P + 1 + ConvectionIntegrator::AddSpecialization<3, 2, 2>(); + ConvectionIntegrator::AddSpecialization<3, 3, 3>(); + ConvectionIntegrator::AddSpecialization<3, 4, 4>(); + ConvectionIntegrator::AddSpecialization<3, 5, 5>(); + ConvectionIntegrator::AddSpecialization<3, 6, 6>(); + // Q = P + 2 + ConvectionIntegrator::AddSpecialization<3, 2, 3>(); + ConvectionIntegrator::AddSpecialization<3, 3, 4>(); + ConvectionIntegrator::AddSpecialization<3, 4, 5>(); + ConvectionIntegrator::AddSpecialization<3, 5, 6>(); + ConvectionIntegrator::AddSpecialization<3, 6, 7>(); +} + // PA Convection Assemble 2D kernel static void PAConvectionSetup2D(const int NQ, const int NE, @@ -187,1348 +226,38 @@ void ConvectionIntegrator::AssembleDiagonalPA(Vector &diag) } } -// PA Convection Apply 2D kernel -template static -void PAConvectionApply2D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) +inline ConvectionIntegrator::ApplyKernelType +ConvectionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int) { - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto G = Reshape(g.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); - auto x = Reshape(x_.Read(), D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + if (DIM == 2) { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - - real_t u[max_D1D][max_D1D]; - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - u[dy][dx] = x(dx,dy,e); - } - } - real_t Bu[max_D1D][max_Q1D]; - real_t Gu[max_D1D][max_Q1D]; - for (int dy = 0; dy < D1D; ++dy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - Bu[dy][qx] = 0.0; - Gu[dy][qx] = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t gx = G(qx,dx); - const real_t x = u[dy][dx]; - Bu[dy][qx] += bx * x; - Gu[dy][qx] += gx * x; - } - } - } - real_t GBu[max_Q1D][max_Q1D]; - real_t BGu[max_Q1D][max_Q1D]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - GBu[qy][qx] = 0.0; - BGu[qy][qx] = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - const real_t gx = G(qy,dy); - GBu[qy][qx] += gx * Bu[dy][qx]; - BGu[qy][qx] += bx * Gu[dy][qx]; - } - } - } - // Calculate Dxy, xDy in plane - real_t DGu[max_Q1D][max_Q1D]; - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t O1 = op(qx,qy,0,e); - const real_t O2 = op(qx,qy,1,e); - - const real_t gradX = BGu[qy][qx]; - const real_t gradY = GBu[qy][qx]; - - DGu[qy][qx] = (O1 * gradX) + (O2 * gradY); - } - } - real_t BDGu[max_D1D][max_Q1D]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int dy = 0; dy < D1D; ++dy) - { - BDGu[dy][qx] = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t w = Bt(dy,qy); - BDGu[dy][qx] += w * DGu[qy][qx]; - } - } - } - for (int dx = 0; dx < D1D; ++dx) - { - for (int dy = 0; dy < D1D; ++dy) - { - real_t BBDGu = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t w = Bt(dx,qx); - BBDGu += w * BDGu[dy][qx]; - } - y(dx,dy,e) += BBDGu; - } - } - }); -} - -// Optimized PA Convection Apply 2D kernel -template static -void SmemPAConvectionApply2D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto G = Reshape(g.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); - auto x = Reshape(x_.Read(), D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); - mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e) - { - const int tidz = MFEM_THREAD_ID(z); - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - // constexpr int MDQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D; - MFEM_SHARED real_t u[NBZ][max_D1D][max_D1D]; - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(dx,x,D1D) - { - // e is really equal to e+tidz - u[tidz][dy][dx] = x(dx,dy,e); - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t Bu[NBZ][max_D1D][max_Q1D]; - MFEM_SHARED real_t Gu[NBZ][max_D1D][max_Q1D]; - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - Bu[tidz][dy][qx] = 0.0; - Gu[tidz][dy][qx] = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t gx = G(qx,dx); - const real_t x = u[tidz][dy][dx]; - Bu[tidz][dy][qx] += bx * x; - Gu[tidz][dy][qx] += gx * x; - } - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t GBu[NBZ][max_Q1D][max_Q1D]; - MFEM_SHARED real_t BGu[NBZ][max_Q1D][max_Q1D]; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - GBu[tidz][qy][qx] = 0.0; - BGu[tidz][qy][qx] = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - const real_t gx = G(qy,dy); - GBu[tidz][qy][qx] += gx * Bu[tidz][dy][qx]; - BGu[tidz][qy][qx] += bx * Gu[tidz][dy][qx]; - } - } - } - MFEM_SYNC_THREAD; - // Calculate Dxy, xDy in plane - MFEM_SHARED real_t DGu[NBZ][max_Q1D][max_Q1D]; - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const real_t O1 = op(qx,qy,0,e); - const real_t O2 = op(qx,qy,1,e); - - const real_t gradX = BGu[tidz][qy][qx]; - const real_t gradY = GBu[tidz][qy][qx]; - - DGu[tidz][qy][qx] = (O1 * gradX) + (O2 * gradY); - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t BDGu[NBZ][max_D1D][max_Q1D]; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - BDGu[tidz][dy][qx] = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t w = Bt(dy,qy); - BDGu[tidz][dy][qx] += w * DGu[tidz][qy][qx]; - } - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(dx,x,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - real_t BBDGu = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t w = Bt(dx,qx); - BBDGu += w * BDGu[tidz][dy][qx]; - } - y(dx,dy,e) += BBDGu; - } - } - }); -} - -// PA Convection Apply 3D kernel -template static -void PAConvectionApply3D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto G = Reshape(g.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); - auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - - real_t u[max_D1D][max_D1D][max_D1D]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - u[dz][dy][dx] = x(dx,dy,dz,e); - } - } - } - real_t Bu[max_D1D][max_D1D][max_Q1D]; - real_t Gu[max_D1D][max_D1D][max_Q1D]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int dy = 0; dy < D1D; ++dy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - Bu[dz][dy][qx] = 0.0; - Gu[dz][dy][qx] = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t gx = G(qx,dx); - const real_t x = u[dz][dy][dx]; - Bu[dz][dy][qx] += bx * x; - Gu[dz][dy][qx] += gx * x; - } - } - } - } - real_t BBu[max_D1D][max_Q1D][max_Q1D]; - real_t GBu[max_D1D][max_Q1D][max_Q1D]; - real_t BGu[max_D1D][max_Q1D][max_Q1D]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - BBu[dz][qy][qx] = 0.0; - GBu[dz][qy][qx] = 0.0; - BGu[dz][qy][qx] = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - const real_t gx = G(qy,dy); - BBu[dz][qy][qx] += bx * Bu[dz][dy][qx]; - GBu[dz][qy][qx] += gx * Bu[dz][dy][qx]; - BGu[dz][qy][qx] += bx * Gu[dz][dy][qx]; - } - } - } - } - real_t GBBu[max_Q1D][max_Q1D][max_Q1D]; - real_t BGBu[max_Q1D][max_Q1D][max_Q1D]; - real_t BBGu[max_Q1D][max_Q1D][max_Q1D]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qz = 0; qz < Q1D; ++qz) - { - GBBu[qz][qy][qx] = 0.0; - BGBu[qz][qy][qx] = 0.0; - BBGu[qz][qy][qx] = 0.0; - for (int dz = 0; dz < D1D; ++dz) - { - const real_t bx = B(qz,dz); - const real_t gx = G(qz,dz); - GBBu[qz][qy][qx] += gx * BBu[dz][qy][qx]; - BGBu[qz][qy][qx] += bx * GBu[dz][qy][qx]; - BBGu[qz][qy][qx] += bx * BGu[dz][qy][qx]; - } - } - } - } - // Calculate Dxy, xDy in plane - real_t DGu[max_Q1D][max_Q1D][max_Q1D]; - for (int qz = 0; qz < Q1D; ++qz) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t O1 = op(qx,qy,qz,0,e); - const real_t O2 = op(qx,qy,qz,1,e); - const real_t O3 = op(qx,qy,qz,2,e); - - const real_t gradX = BBGu[qz][qy][qx]; - const real_t gradY = BGBu[qz][qy][qx]; - const real_t gradZ = GBBu[qz][qy][qx]; - - DGu[qz][qy][qx] = (O1 * gradX) + (O2 * gradY) + (O3 * gradZ); - } - } - } - real_t BDGu[max_D1D][max_Q1D][max_Q1D]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int dz = 0; dz < D1D; ++dz) - { - BDGu[dz][qy][qx] = 0.0; - for (int qz = 0; qz < Q1D; ++qz) - { - const real_t w = Bt(dz,qz); - BDGu[dz][qy][qx] += w * DGu[qz][qy][qx]; - } - } - } - } - real_t BBDGu[max_D1D][max_D1D][max_Q1D]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int qx = 0; qx < Q1D; ++qx) - { - for (int dy = 0; dy < D1D; ++dy) - { - BBDGu[dz][dy][qx] = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t w = Bt(dy,qy); - BBDGu[dz][dy][qx] += w * BDGu[dz][qy][qx]; - } - } - } - } - for (int dz = 0; dz < D1D; ++dz) - { - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - real_t BBBDGu = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t w = Bt(dx,qx); - BBBDGu += w * BBDGu[dz][dy][qx]; - } - y(dx,dy,dz,e) += BBBDGu; - } - } - } - }); -} - -// Optimized PA Convection Apply 3D kernel -template static -void SmemPAConvectionApply3D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto G = Reshape(g.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); - auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); - mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - constexpr int max_DQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D; - MFEM_SHARED real_t sm0[max_DQ*max_DQ*max_DQ]; - MFEM_SHARED real_t sm1[max_DQ*max_DQ*max_DQ]; - MFEM_SHARED real_t sm2[max_DQ*max_DQ*max_DQ]; - MFEM_SHARED real_t sm3[max_DQ*max_DQ*max_DQ]; - MFEM_SHARED real_t sm4[max_DQ*max_DQ*max_DQ]; - MFEM_SHARED real_t sm5[max_DQ*max_DQ*max_DQ]; - - real_t (*u)[max_D1D][max_D1D] = (real_t (*)[max_D1D][max_D1D]) sm0; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(dx,x,D1D) - { - u[dz][dy][dx] = x(dx,dy,dz,e); - } - } - } - MFEM_SYNC_THREAD; - real_t (*Bu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm1; - real_t (*Gu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm2; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - real_t Bu_ = 0.0; - real_t Gu_ = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t gx = G(qx,dx); - const real_t x = u[dz][dy][dx]; - Bu_ += bx * x; - Gu_ += gx * x; - } - Bu[dz][dy][qx] = Bu_; - Gu[dz][dy][qx] = Gu_; - } - } - } - MFEM_SYNC_THREAD; - real_t (*BBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm3; - real_t (*GBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm4; - real_t (*BGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm5; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - real_t BBu_ = 0.0; - real_t GBu_ = 0.0; - real_t BGu_ = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - const real_t gx = G(qy,dy); - BBu_ += bx * Bu[dz][dy][qx]; - GBu_ += gx * Bu[dz][dy][qx]; - BGu_ += bx * Gu[dz][dy][qx]; - } - BBu[dz][qy][qx] = BBu_; - GBu[dz][qy][qx] = GBu_; - BGu[dz][qy][qx] = BGu_; - } - } - } - MFEM_SYNC_THREAD; - real_t (*GBBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm0; - real_t (*BGBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm1; - real_t (*BBGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm2; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qz,z,Q1D) - { - real_t GBBu_ = 0.0; - real_t BGBu_ = 0.0; - real_t BBGu_ = 0.0; - for (int dz = 0; dz < D1D; ++dz) - { - const real_t bx = B(qz,dz); - const real_t gx = G(qz,dz); - GBBu_ += gx * BBu[dz][qy][qx]; - BGBu_ += bx * GBu[dz][qy][qx]; - BBGu_ += bx * BGu[dz][qy][qx]; - } - GBBu[qz][qy][qx] = GBBu_; - BGBu[qz][qy][qx] = BGBu_; - BBGu[qz][qy][qx] = BBGu_; - } - } - } - MFEM_SYNC_THREAD; - real_t (*DGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm3; - MFEM_FOREACH_THREAD(qz,z,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const real_t O1 = op(qx,qy,qz,0,e); - const real_t O2 = op(qx,qy,qz,1,e); - const real_t O3 = op(qx,qy,qz,2,e); - - const real_t gradX = BBGu[qz][qy][qx]; - const real_t gradY = BGBu[qz][qy][qx]; - const real_t gradZ = GBBu[qz][qy][qx]; - - DGu[qz][qy][qx] = (O1 * gradX) + (O2 * gradY) + (O3 * gradZ); - } - } - } - MFEM_SYNC_THREAD; - real_t (*BDGu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm4; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(dz,z,D1D) - { - real_t BDGu_ = 0.0; - for (int qz = 0; qz < Q1D; ++qz) - { - const real_t w = Bt(dz,qz); - BDGu_ += w * DGu[qz][qy][qx]; - } - BDGu[dz][qy][qx] = BDGu_; - } - } - } - MFEM_SYNC_THREAD; - real_t (*BBDGu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm5; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - real_t BBDGu_ = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t w = Bt(dy,qy); - BBDGu_ += w * BDGu[dz][qy][qx]; - } - BBDGu[dz][dy][qx] = BBDGu_; - } - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(dx,x,D1D) - { - real_t BBBDGu = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t w = Bt(dx,qx); - BBBDGu += w * BBDGu[dz][dy][qx]; - } - y(dx,dy,dz,e) += BBBDGu; - } - } - } - }); -} - -// PA Convection Apply 2D kernel -template static -void PAConvectionApplyT2D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto Gt = Reshape(gt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); - auto x = Reshape(x_.Read(), D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - - real_t u[max_D1D][max_D1D]; - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - u[dy][dx] = x(dx,dy,e); - } - } - real_t Bu[max_D1D][max_Q1D]; - for (int dy = 0; dy < D1D; ++dy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - Bu[dy][qx] = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t x = u[dy][dx]; - Bu[dy][qx] += bx * x; - } - } - } - real_t BBu[max_Q1D][max_Q1D]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - BBu[qy][qx] = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - BBu[qy][qx] += bx * Bu[dy][qx]; - } - } - } - // Calculate Dxy, xDy in plane - real_t DBu[max_Q1D][max_Q1D][2]; - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t O1 = op(qx,qy,0,e); - const real_t O2 = op(qx,qy,1,e); - - const real_t X = BBu[qy][qx]; - - DBu[qy][qx][0] = O1 * X; - DBu[qy][qx][1] = O2 * X; - } - } - real_t GDBu[max_D1D][max_Q1D][2]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int dy = 0; dy < D1D; ++dy) - { - GDBu[dy][qx][0] = 0.0; - GDBu[dy][qx][1] = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t by = Bt(dy,qy); - const real_t gy = Gt(dy,qy); - GDBu[dy][qx][0] += by * DBu[qy][qx][0]; - GDBu[dy][qx][1] += gy * DBu[qy][qx][1]; - } - } - } - for (int dx = 0; dx < D1D; ++dx) - { - for (int dy = 0; dy < D1D; ++dy) - { - real_t res = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t bx = Bt(dx,qx); - const real_t gx = Gt(dx,qx); - res += gx * GDBu[dy][qx][0] + bx * GDBu[dy][qx][1]; - } - y(dx,dy,e) += res; - } - } - }); -} - -// Optimized PA Convection Apply 2D kernel -template static -void SmemPAConvectionApplyT2D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto Gt = Reshape(gt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE); - auto x = Reshape(x_.Read(), D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE); - mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e) - { - const int tidz = MFEM_THREAD_ID(z); - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - MFEM_SHARED real_t u[NBZ][max_D1D][max_D1D]; - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(dx,x,D1D) - { - // e is really equal to e+tidz - u[tidz][dy][dx] = x(dx,dy,e); - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t Bu[NBZ][max_D1D][max_Q1D]; - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - Bu[tidz][dy][qx] = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t x = u[tidz][dy][dx]; - Bu[tidz][dy][qx] += bx * x; - } - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t BBu[NBZ][max_Q1D][max_Q1D]; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - BBu[tidz][qy][qx] = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - BBu[tidz][qy][qx] += bx * Bu[tidz][dy][qx]; - } - } - } - MFEM_SYNC_THREAD; - // Calculate Dxy, xDy in plane - MFEM_SHARED real_t DBu[NBZ][max_Q1D][max_Q1D][2]; - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const real_t O1 = op(qx,qy,0,e); - const real_t O2 = op(qx,qy,1,e); - - const real_t X = BBu[tidz][qy][qx]; - - DBu[tidz][qy][qx][0] = O1 * X; - DBu[tidz][qy][qx][1] = O2 * X; - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t GDBu[NBZ][max_D1D][max_Q1D][2]; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - GDBu[tidz][dy][qx][0] = 0.0; - GDBu[tidz][dy][qx][1] = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t by = Bt(dy,qy); - const real_t gy = Gt(dy,qy); - GDBu[tidz][dy][qx][0] += by * DBu[tidz][qy][qx][0]; - GDBu[tidz][dy][qx][1] += gy * DBu[tidz][qy][qx][1]; - } - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(dx,x,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - real_t res = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t bx = Bt(dx,qx); - const real_t gx = Gt(dx,qx); - res += gx * GDBu[tidz][dy][qx][0] + bx * GDBu[tidz][dy][qx][1]; - } - y(dx,dy,e) += res; - } - } - }); -} - -// PA Convection Apply 3D kernel -template static -void PAConvectionApplyT3D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto Gt = Reshape(gt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); - auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - - real_t u[max_D1D][max_D1D][max_D1D]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - u[dz][dy][dx] = x(dx,dy,dz,e); - } - } - } - real_t Bu[max_D1D][max_D1D][max_Q1D]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int dy = 0; dy < D1D; ++dy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - Bu[dz][dy][qx] = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t x = u[dz][dy][dx]; - Bu[dz][dy][qx] += bx * x; - } - } - } - } - real_t BBu[max_D1D][max_Q1D][max_Q1D]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - BBu[dz][qy][qx] = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - BBu[dz][qy][qx] += bx * Bu[dz][dy][qx]; - } - } - } - } - real_t BBBu[max_Q1D][max_Q1D][max_Q1D]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qz = 0; qz < Q1D; ++qz) - { - BBBu[qz][qy][qx] = 0.0; - for (int dz = 0; dz < D1D; ++dz) - { - const real_t bx = B(qz,dz); - BBBu[qz][qy][qx] += bx * BBu[dz][qy][qx]; - } - } - } - } - // Calculate Dxy, xDy in plane - real_t DBu[max_Q1D][max_Q1D][max_Q1D][3]; - for (int qz = 0; qz < Q1D; ++qz) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t O1 = op(qx,qy,qz,0,e); - const real_t O2 = op(qx,qy,qz,1,e); - const real_t O3 = op(qx,qy,qz,2,e); - - const real_t X = BBBu[qz][qy][qx]; - - DBu[qz][qy][qx][0] = O1 * X; - DBu[qz][qy][qx][1] = O2 * X; - DBu[qz][qy][qx][2] = O3 * X; - } - } - } - real_t GDBu[max_D1D][max_Q1D][max_Q1D][3]; - for (int qx = 0; qx < Q1D; ++qx) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int dz = 0; dz < D1D; ++dz) - { - GDBu[dz][qy][qx][0] = 0.0; - GDBu[dz][qy][qx][1] = 0.0; - GDBu[dz][qy][qx][2] = 0.0; - for (int qz = 0; qz < Q1D; ++qz) - { - const real_t bz = Bt(dz,qz); - const real_t gz = Gt(dz,qz); - GDBu[dz][qy][qx][0] += bz * DBu[qz][qy][qx][0]; - GDBu[dz][qy][qx][1] += bz * DBu[qz][qy][qx][1]; - GDBu[dz][qy][qx][2] += gz * DBu[qz][qy][qx][2]; - } - } - } - } - real_t GGDBu[max_D1D][max_D1D][max_Q1D][3]; - for (int dz = 0; dz < D1D; ++dz) - { - for (int qx = 0; qx < Q1D; ++qx) - { - for (int dy = 0; dy < D1D; ++dy) - { - GGDBu[dz][dy][qx][0] = 0.0; - GGDBu[dz][dy][qx][1] = 0.0; - GGDBu[dz][dy][qx][2] = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t by = Bt(dy,qy); - const real_t gy = Gt(dy,qy); - GGDBu[dz][dy][qx][0] += by * GDBu[dz][qy][qx][0]; - GGDBu[dz][dy][qx][1] += gy * GDBu[dz][qy][qx][1]; - GGDBu[dz][dy][qx][2] += by * GDBu[dz][qy][qx][2]; - } - } - } - } - for (int dz = 0; dz < D1D; ++dz) - { - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - real_t res = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t bx = Bt(dx,qx); - const real_t gx = Gt(dx,qx); - res += gx * GGDBu[dz][dy][qx][0]; - res += bx * GGDBu[dz][dy][qx][1]; - res += bx * GGDBu[dz][dy][qx][2]; - } - y(dx,dy,dz,e) += res; - } - } - } - }); -} - -// Optimized PA Convection Apply 3D kernel -template static -void SmemPAConvectionApplyT3D(const int ne, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int NE = ne; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto Gt = Reshape(gt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE); - auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE); - mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - constexpr int max_DQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D; - MFEM_SHARED real_t sm0[3*max_DQ*max_DQ*max_DQ]; - MFEM_SHARED real_t sm1[3*max_DQ*max_DQ*max_DQ]; - - real_t (*u)[max_D1D][max_D1D] = (real_t (*)[max_D1D][max_D1D]) sm0; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(dx,x,D1D) - { - u[dz][dy][dx] = x(dx,dy,dz,e); - } - } - } - MFEM_SYNC_THREAD; - real_t (*Bu)[max_D1D][max_Q1D] = (real_t (*)[max_D1D][max_Q1D])sm1; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - real_t Bu_ = 0.0; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t bx = B(qx,dx); - const real_t x = u[dz][dy][dx]; - Bu_ += bx * x; - } - Bu[dz][dy][qx] = Bu_; - } - } - } - MFEM_SYNC_THREAD; - real_t (*BBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm0; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - real_t BBu_ = 0.0; - for (int dy = 0; dy < D1D; ++dy) - { - const real_t bx = B(qy,dy); - BBu_ += bx * Bu[dz][dy][qx]; - } - BBu[dz][qy][qx] = BBu_; - } - } - } - MFEM_SYNC_THREAD; - real_t (*BBBu)[max_Q1D][max_Q1D] = (real_t (*)[max_Q1D][max_Q1D])sm1; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qz,z,Q1D) - { - real_t BBBu_ = 0.0; - for (int dz = 0; dz < D1D; ++dz) - { - const real_t bx = B(qz,dz); - BBBu_ += bx * BBu[dz][qy][qx]; - } - BBBu[qz][qy][qx] = BBBu_; - } - } - } - MFEM_SYNC_THREAD; - real_t (*DBu)[max_Q1D][max_Q1D][3] = (real_t (*)[max_Q1D][max_Q1D][3])sm0; - MFEM_FOREACH_THREAD(qz,z,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const real_t O1 = op(qx,qy,qz,0,e); - const real_t O2 = op(qx,qy,qz,1,e); - const real_t O3 = op(qx,qy,qz,2,e); - - const real_t X = BBBu[qz][qy][qx]; - - DBu[qz][qy][qx][0] = O1 * X; - DBu[qz][qy][qx][1] = O2 * X; - DBu[qz][qy][qx][2] = O3 * X; - } - } - } - MFEM_SYNC_THREAD; - real_t (*GDBu)[max_Q1D][max_Q1D][3] = (real_t (*)[max_Q1D][max_Q1D][3])sm1; - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - MFEM_FOREACH_THREAD(dz,z,D1D) - { - real_t GDBu0 = 0.0; - real_t GDBu1 = 0.0; - real_t GDBu2 = 0.0; - for (int qz = 0; qz < Q1D; ++qz) - { - const real_t bz = Bt(dz,qz); - const real_t gz = Gt(dz,qz); - GDBu0 += bz * DBu[qz][qy][qx][0]; - GDBu1 += bz * DBu[qz][qy][qx][1]; - GDBu2 += gz * DBu[qz][qy][qx][2]; - } - GDBu[dz][qy][qx][0] = GDBu0; - GDBu[dz][qy][qx][1] = GDBu1; - GDBu[dz][qy][qx][2] = GDBu2; - } - } - } - MFEM_SYNC_THREAD; - real_t (*GGDBu)[max_D1D][max_Q1D][3] = (real_t (*)[max_D1D][max_Q1D][3])sm0; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - real_t GGDBu0 = 0.0; - real_t GGDBu1 = 0.0; - real_t GGDBu2 = 0.0; - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t by = Bt(dy,qy); - const real_t gy = Gt(dy,qy); - GGDBu0 += by * GDBu[dz][qy][qx][0]; - GGDBu1 += gy * GDBu[dz][qy][qx][1]; - GGDBu2 += by * GDBu[dz][qy][qx][2]; - } - GGDBu[dz][dy][qx][0] = GGDBu0; - GGDBu[dz][dy][qx][1] = GGDBu1; - GGDBu[dz][dy][qx][2] = GGDBu2; - } - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(dz,z,D1D) - { - MFEM_FOREACH_THREAD(dy,y,D1D) - { - MFEM_FOREACH_THREAD(dx,x,D1D) - { - real_t res = 0.0; - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t bx = Bt(dx,qx); - const real_t gx = Gt(dx,qx); - res += gx * GGDBu[dz][dy][qx][0]; - res += bx * GGDBu[dz][dy][qx][1]; - res += bx * GGDBu[dz][dy][qx][2]; - } - y(dx,dy,dz,e) += res; - } - } - } - }); -} - -static void PAConvectionApply(const int dim, - const int D1D, - const int Q1D, - const int NE, - const Array &B, - const Array &G, - const Array &Bt, - const Array &Gt, - const Vector &op, - const Vector &x, - Vector &y) -{ - if (dim == 2) - { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return SmemPAConvectionApply2D<2,2,8>(NE,B,G,Bt,Gt,op,x,y); - case 0x33: return SmemPAConvectionApply2D<3,3,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x34: return SmemPAConvectionApply2D<3,4,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x44: return SmemPAConvectionApply2D<4,4,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x46: return SmemPAConvectionApply2D<4,6,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x55: return SmemPAConvectionApply2D<5,5,2>(NE,B,G,Bt,Gt,op,x,y); - case 0x58: return SmemPAConvectionApply2D<5,8,2>(NE,B,G,Bt,Gt,op,x,y); - case 0x66: return SmemPAConvectionApply2D<6,6,1>(NE,B,G,Bt,Gt,op,x,y); - case 0x77: return SmemPAConvectionApply2D<7,7,1>(NE,B,G,Bt,Gt,op,x,y); - case 0x88: return SmemPAConvectionApply2D<8,8,1>(NE,B,G,Bt,Gt,op,x,y); - case 0x99: return SmemPAConvectionApply2D<9,9,1>(NE,B,G,Bt,Gt,op,x,y); - default: return PAConvectionApply2D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D); - } + return PAConvectionApply2D; } - else if (dim == 3) + else if (DIM == 3) { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return SmemPAConvectionApply3D<2,2>(NE,B,G,Bt,Gt,op,x,y); - case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y); - case 0x24: return SmemPAConvectionApply3D<2,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x26: return SmemPAConvectionApply3D<2,6>(NE,B,G,Bt,Gt,op,x,y); - case 0x34: return SmemPAConvectionApply3D<3,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x35: return SmemPAConvectionApply3D<3,5>(NE,B,G,Bt,Gt,op,x,y); - case 0x45: return SmemPAConvectionApply3D<4,5>(NE,B,G,Bt,Gt,op,x,y); - case 0x48: return SmemPAConvectionApply3D<4,8>(NE,B,G,Bt,Gt,op,x,y); - case 0x56: return SmemPAConvectionApply3D<5,6>(NE,B,G,Bt,Gt,op,x,y); - case 0x67: return SmemPAConvectionApply3D<6,7>(NE,B,G,Bt,Gt,op,x,y); - case 0x78: return SmemPAConvectionApply3D<7,8>(NE,B,G,Bt,Gt,op,x,y); - case 0x89: return SmemPAConvectionApply3D<8,9>(NE,B,G,Bt,Gt,op,x,y); - default: return PAConvectionApply3D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D); - } + return PAConvectionApply3D; + } + else + { + MFEM_ABORT(""); } - MFEM_ABORT("Unknown kernel."); } -static void PAConvectionApplyT(const int dim, - const int D1D, - const int Q1D, - const int NE, - const Array &B, - const Array &G, - const Array &Bt, - const Array &Gt, - const Vector &op, - const Vector &x, - Vector &y) +inline ConvectionIntegrator::ApplyKernelType +ConvectionIntegrator::ApplyPATKernels::Fallback(int DIM, int, int) { - if (dim == 2) + if (DIM == 2) { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return SmemPAConvectionApplyT2D<2,2,8>(NE,B,G,Bt,Gt,op,x,y); - case 0x33: return SmemPAConvectionApplyT2D<3,3,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x34: return SmemPAConvectionApplyT2D<3,4,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x44: return SmemPAConvectionApplyT2D<4,4,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x46: return SmemPAConvectionApplyT2D<4,6,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x55: return SmemPAConvectionApplyT2D<5,5,2>(NE,B,G,Bt,Gt,op,x,y); - case 0x58: return SmemPAConvectionApplyT2D<5,8,2>(NE,B,G,Bt,Gt,op,x,y); - case 0x66: return SmemPAConvectionApplyT2D<6,6,1>(NE,B,G,Bt,Gt,op,x,y); - case 0x77: return SmemPAConvectionApplyT2D<7,7,1>(NE,B,G,Bt,Gt,op,x,y); - case 0x88: return SmemPAConvectionApplyT2D<8,8,1>(NE,B,G,Bt,Gt,op,x,y); - case 0x99: return SmemPAConvectionApplyT2D<9,9,1>(NE,B,G,Bt,Gt,op,x,y); - default: return PAConvectionApplyT2D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D); - } + return PAConvectionApplyT2D; } - else if (dim == 3) + else if (DIM == 3) { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return SmemPAConvectionApplyT3D<2,2>(NE,B,G,Bt,Gt,op,x,y); - case 0x23: return SmemPAConvectionApplyT3D<2,3>(NE,B,G,Bt,Gt,op,x,y); - case 0x24: return SmemPAConvectionApplyT3D<2,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x26: return SmemPAConvectionApplyT3D<2,6>(NE,B,G,Bt,Gt,op,x,y); - case 0x34: return SmemPAConvectionApplyT3D<3,4>(NE,B,G,Bt,Gt,op,x,y); - case 0x35: return SmemPAConvectionApplyT3D<3,5>(NE,B,G,Bt,Gt,op,x,y); - case 0x45: return SmemPAConvectionApplyT3D<4,5>(NE,B,G,Bt,Gt,op,x,y); - case 0x48: return SmemPAConvectionApplyT3D<4,8>(NE,B,G,Bt,Gt,op,x,y); - case 0x56: return SmemPAConvectionApplyT3D<5,6>(NE,B,G,Bt,Gt,op,x,y); - case 0x67: return SmemPAConvectionApplyT3D<6,7>(NE,B,G,Bt,Gt,op,x,y); - case 0x78: return SmemPAConvectionApplyT3D<7,8>(NE,B,G,Bt,Gt,op,x,y); - case 0x89: return SmemPAConvectionApplyT3D<8,9>(NE,B,G,Bt,Gt,op,x,y); - default: return PAConvectionApplyT3D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D); - } + return PAConvectionApplyT3D; + } + else + { + MFEM_ABORT(""); } - MFEM_ABORT("Unknown kernel."); } void ConvectionIntegrator::AddMultPA(const Vector &x, Vector &y) const @@ -1539,9 +268,8 @@ void ConvectionIntegrator::AddMultPA(const Vector &x, Vector &y) const } else { - PAConvectionApply(dim, dofs1D, quad1D, ne, - maps->B, maps->G, maps->Bt, maps->Gt, - pa_data, x, y); + ApplyPAKernels::Run(dim, dofs1D, quad1D, ne, maps->B, maps->G, maps->Bt, + maps->Gt, pa_data, x, y, dofs1D, quad1D); } } @@ -1554,10 +282,10 @@ void ConvectionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const } else { - PAConvectionApplyT(dim, dofs1D, quad1D, ne, - maps->B, maps->G, maps->Bt, maps->Gt, - pa_data, x, y); + ApplyPATKernels::Run(dim, dofs1D, quad1D, ne, maps->B, maps->G, maps->Bt, + maps->Gt, pa_data, x, y, dofs1D, quad1D); } } } // namespace mfem +/// \endcond DO_NOT_DOCUMENT diff --git a/fem/integ/bilininteg_curlcurl_pa.cpp b/fem/integ/bilininteg_curlcurl_pa.cpp index 06374681ee..2562befc6f 100644 --- a/fem/integ/bilininteg_curlcurl_pa.cpp +++ b/fem/integ/bilininteg_curlcurl_pa.cpp @@ -15,6 +15,86 @@ namespace mfem { +CurlCurlIntegrator::CurlCurlIntegrator() : Q(nullptr), DQ(nullptr), MQ(nullptr) +{ + static Kernels kernels; +} + +CurlCurlIntegrator::CurlCurlIntegrator(Coefficient &q, + const IntegrationRule *ir) + : BilinearFormIntegrator(ir), Q(&q), DQ(nullptr), MQ(nullptr) +{ + static Kernels kernels; +} + +CurlCurlIntegrator::CurlCurlIntegrator(DiagonalMatrixCoefficient &dq, + const IntegrationRule *ir) + : BilinearFormIntegrator(ir), Q(nullptr), DQ(&dq), MQ(nullptr) +{ + static Kernels kernels; +} + +CurlCurlIntegrator::CurlCurlIntegrator(MatrixCoefficient &mq, + const IntegrationRule *ir) + : BilinearFormIntegrator(ir), Q(nullptr), DQ(nullptr), MQ(&mq) +{ + static Kernels kernels; +} + +/// \cond DO_NOT_DOCUMENT + +CurlCurlIntegrator::Kernels::Kernels() +{ + CurlCurlIntegrator::AddSpecialization<3, 2, 3>(); + CurlCurlIntegrator::AddSpecialization<3, 3, 4>(); + CurlCurlIntegrator::AddSpecialization<3, 4, 5>(); + CurlCurlIntegrator::AddSpecialization<3, 5, 6>(); +} + +CurlCurlIntegrator::ApplyKernelType +CurlCurlIntegrator::ApplyPAKernels::Fallback(int DIM, int, int) +{ + if (DIM == 2) { return internal::PACurlCurlApply2D; } + else if (DIM == 3) + { + if (Device::Allows(Backend::DEVICE_MASK)) + { + return internal::SmemPACurlCurlApply3D; + } + else + { + return internal::PACurlCurlApply3D; + } + } + else { MFEM_ABORT(""); } +} + +CurlCurlIntegrator::DiagonalKernelType +CurlCurlIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int) +{ + if (DIM == 2) + { + return internal::PACurlCurlAssembleDiagonal2D; + } + else if (DIM == 3) + { + if (Device::Allows(Backend::DEVICE_MASK)) + { + return internal::SmemPACurlCurlAssembleDiagonal3D; + } + else + { + return internal::PACurlCurlAssembleDiagonal3D; + } + } + else + { + MFEM_ABORT(""); + } +} + +/// \endcond DO_NOT_DOCUMENT + void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes) { // Assumes tensor-product elements @@ -77,129 +157,16 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes) void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag) { - if (dim == 3) - { - if (Device::Allows(Backend::DEVICE_MASK)) - { - const int ID = (dofs1D << 4) | quad1D; - switch (ID) - { - case 0x23: - return internal::SmemPACurlCurlAssembleDiagonal3D<2,3>( - dofs1D, - quad1D, - symmetric, ne, - mapsO->B, mapsC->B, - mapsO->G, mapsC->G, - pa_data, diag); - case 0x34: - return internal::SmemPACurlCurlAssembleDiagonal3D<3,4>( - dofs1D, - quad1D, - symmetric, ne, - mapsO->B, mapsC->B, - mapsO->G, mapsC->G, - pa_data, diag); - case 0x45: - return internal::SmemPACurlCurlAssembleDiagonal3D<4,5>( - dofs1D, - quad1D, - symmetric, ne, - mapsO->B, mapsC->B, - mapsO->G, mapsC->G, - pa_data, diag); - case 0x56: - return internal::SmemPACurlCurlAssembleDiagonal3D<5,6>( - dofs1D, - quad1D, - symmetric, ne, - mapsO->B, mapsC->B, - mapsO->G, mapsC->G, - pa_data, diag); - default: - return internal::SmemPACurlCurlAssembleDiagonal3D( - dofs1D, quad1D, - symmetric, ne, - mapsO->B, mapsC->B, - mapsO->G, mapsC->G, - pa_data, diag); - } - } - else - { - internal::PACurlCurlAssembleDiagonal3D(dofs1D, quad1D, symmetric, ne, - mapsO->B, mapsC->B, - mapsO->G, mapsC->G, - pa_data, diag); - } - } - else if (dim == 2) - { - internal::PACurlCurlAssembleDiagonal2D(dofs1D, quad1D, ne, - mapsO->B, mapsC->G, pa_data, diag); - } - else - { - MFEM_ABORT("Unsupported dimension!"); - } + DiagonalPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne, + mapsO->B, mapsC->B, mapsO->G, mapsC->G, pa_data, + diag); } void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const { - if (dim == 3) - { - if (Device::Allows(Backend::DEVICE_MASK)) - { - const int ID = (dofs1D << 4) | quad1D; - switch (ID) - { - case 0x23: - return internal::SmemPACurlCurlApply3D<2,3>( - dofs1D, quad1D, - symmetric, ne, - mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, - mapsC->G, mapsC->Gt, pa_data, x, y); - case 0x34: - return internal::SmemPACurlCurlApply3D<3,4>( - dofs1D, quad1D, - symmetric, ne, - mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, - mapsC->G, mapsC->Gt, pa_data, x, y); - case 0x45: - return internal::SmemPACurlCurlApply3D<4,5>( - dofs1D, quad1D, - symmetric, ne, - mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, - mapsC->G, mapsC->Gt, pa_data, x, y); - case 0x56: - return internal::SmemPACurlCurlApply3D<5,6>( - dofs1D, quad1D, - symmetric, ne, - mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, - mapsC->G, mapsC->Gt, pa_data, x, y); - default: - return internal::SmemPACurlCurlApply3D( - dofs1D, quad1D, symmetric, ne, - mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, - mapsC->G, mapsC->Gt, pa_data, x, y); - } - } - else - { - internal::PACurlCurlApply3D(dofs1D, quad1D, symmetric, ne, mapsO->B, mapsC->B, - mapsO->Bt, mapsC->Bt, mapsC->G, mapsC->Gt, - pa_data, x, y); - } - } - else if (dim == 2) - { - internal::PACurlCurlApply2D(dofs1D, quad1D, ne, mapsO->B, mapsO->Bt, - mapsC->G, mapsC->Gt, pa_data, x, y); - } - else - { - MFEM_ABORT("Unsupported dimension!"); - } + ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne, + mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->G, + mapsC->Gt, pa_data, x, y, false); } void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const @@ -209,61 +176,9 @@ void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const auto absO = mapsO->Abs(); auto absC = mapsC->Abs(); - if (dim == 3) - { - if (Device::Allows(Backend::DEVICE_MASK)) - { - const int ID = (dofs1D << 4) | quad1D; - switch (ID) - { - case 0x23: - return internal::SmemPACurlCurlApply3D<2,3>( - dofs1D, quad1D, - symmetric, ne, - absO.B, absC.B, absO.Bt, absC.Bt, - absC.G, absC.Gt, abs_pa_data, x, y, true); - case 0x34: - return internal::SmemPACurlCurlApply3D<3,4>( - dofs1D, quad1D, - symmetric, ne, - absO.B, absC.B, absO.Bt, absC.Bt, - absC.G, absC.Gt, abs_pa_data, x, y, true); - case 0x45: - return internal::SmemPACurlCurlApply3D<4,5>( - dofs1D, quad1D, - symmetric, ne, - absO.B, absC.B, absO.Bt, absC.Bt, - absC.G, absC.Gt, abs_pa_data, x, y, true); - case 0x56: - return internal::SmemPACurlCurlApply3D<5,6>( - dofs1D, quad1D, - symmetric, ne, - absO.B, absC.B, absO.Bt, absC.Bt, - absC.G, absC.Gt, abs_pa_data, x, y, true); - default: - return internal::SmemPACurlCurlApply3D<0,0>( - dofs1D, quad1D, symmetric, ne, - absO.B, absC.B, absO.Bt, absC.Bt, - absC.G, absC.Gt, abs_pa_data, x, y, true); - } - } - else - { - internal::PACurlCurlApply3D<0,0>( - dofs1D, quad1D, symmetric, ne, - absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt, - abs_pa_data, x, y, true); - } - } - else if (dim == 2) - { - internal::PACurlCurlApply2D(dofs1D, quad1D, ne, absO.B, absO.Bt, - absC.G, absC.Gt, abs_pa_data, x, y, true); - } - else - { - MFEM_ABORT("Unsupported dimension!"); - } + ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne, + absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt, + abs_pa_data, x, y, true); } } // namespace mfem diff --git a/fem/integ/bilininteg_dgdiffusion_kernels.hpp b/fem/integ/bilininteg_dgdiffusion_kernels.hpp new file mode 100644 index 0000000000..7cb458e5e1 --- /dev/null +++ b/fem/integ/bilininteg_dgdiffusion_kernels.hpp @@ -0,0 +1,500 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP +#define MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP + +#include "../../general/forall.hpp" +#include "../../mesh/face_nbr_geom.hpp" +#include "../fe/face_map_utils.hpp" +#include "../gridfunc.hpp" +#include "../qfunction.hpp" + +/// \cond DO_NOT_DOCUMENT +namespace mfem +{ + +namespace internal +{ + +template +static void PADGDiffusionApply2D(const int NF, const Array &b, + const Array &bt, + const Array &g, + const Array >, const real_t sigma, + const Vector &pa_data, const Vector &x_, + const Vector &dxdn_, Vector &y_, Vector &dydn_, + const int d1d = 0, const int q1d = 0) +{ + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + + auto B_ = Reshape(b.Read(), Q1D, D1D); + auto G_ = Reshape(g.Read(), Q1D, D1D); + + auto pa = + Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11) + + auto x = Reshape(x_.Read(), D1D, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, 2, NF); + auto dxdn = Reshape(dxdn_.Read(), D1D, 2, NF); + auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF); + + const int NBX = std::max(D1D, Q1D); + + mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void + { + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + + MFEM_SHARED real_t u0[max_D1D]; + MFEM_SHARED real_t u1[max_D1D]; + MFEM_SHARED real_t du0[max_D1D]; + MFEM_SHARED real_t du1[max_D1D]; + + MFEM_SHARED real_t Bu0[max_Q1D]; + MFEM_SHARED real_t Bu1[max_Q1D]; + MFEM_SHARED real_t Bdu0[max_Q1D]; + MFEM_SHARED real_t Bdu1[max_Q1D]; + + MFEM_SHARED real_t r[max_Q1D]; + + MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D]; + DeviceMatrix B(BG, Q1D, D1D); + DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D); + + if (MFEM_THREAD_ID(y) == 0) + { + MFEM_FOREACH_THREAD(p, x, Q1D) + { + for (int d = 0; d < D1D; ++d) + { + B(p, d) = B_(p, d); + G(p, d) = G_(p, d); + } + } + } + MFEM_SYNC_THREAD; + + // copy edge values to u0, u1 and copy edge normals to du0, du1 + MFEM_FOREACH_THREAD(side, y, 2) + { + real_t *u = (side == 0) ? u0 : u1; + real_t *du = (side == 0) ? du0 : du1; + MFEM_FOREACH_THREAD(d, x, D1D) + { + u[d] = x(d, side, f); + du[d] = dxdn(d, side, f); + } + } + MFEM_SYNC_THREAD; + + // eval @ quad points + MFEM_FOREACH_THREAD(side, y, 2) + { + real_t *u = (side == 0) ? u0 : u1; + real_t *du = (side == 0) ? du0 : du1; + real_t *Bu = (side == 0) ? Bu0 : Bu1; + real_t *Bdu = (side == 0) ? Bdu0 : Bdu1; + + MFEM_FOREACH_THREAD(p, x, Q1D) + { + const real_t Je_side[] = {pa(2 + 2 * side, p, f), + pa(2 + 2 * side + 1, p, f) + }; + + Bu[p] = 0.0; + Bdu[p] = 0.0; + + for (int d = 0; d < D1D; ++d) + { + const real_t b = B(p, d); + const real_t g = G(p, d); + + Bu[p] += b * u[d]; + Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d]; + } + } + } + MFEM_SYNC_THREAD; + + // term - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >: + if (MFEM_THREAD_ID(y) == 0) + { + MFEM_FOREACH_THREAD(p, x, Q1D) + { + const real_t q = pa(0, p, f); + const real_t hi = pa(1, p, f); + const real_t jump = Bu0[p] - Bu1[p]; + const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J) + r[p] = -avg + hi * q * jump; + } + } + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD(d, x, D1D) + { + real_t Br = 0.0; + + for (int p = 0; p < Q1D; ++p) + { + Br += B(p, d) * r[p]; + } + + u0[d] = Br; // overwrite u0, u1 + u1[d] = -Br; + } // for d + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD(side, y, 2) + { + real_t *du = (side == 0) ? du0 : du1; + MFEM_FOREACH_THREAD(d, x, D1D) { du[d] = 0.0; } + } + MFEM_SYNC_THREAD; + + // term sigma * < [u], {Q dv/dn} > + MFEM_FOREACH_THREAD(side, y, 2) + { + real_t *const du = (side == 0) ? du0 : du1; + real_t *const u = (side == 0) ? u0 : u1; + + MFEM_FOREACH_THREAD(d, x, D1D) + { + for (int p = 0; p < Q1D; ++p) + { + const real_t Je[] = {pa(2 + 2 * side, p, f), + pa(2 + 2 * side + 1, p, f) + }; + const real_t jump = Bu0[p] - Bu1[p]; + const real_t r_p = Je[0] * jump; // normal + const real_t w_p = Je[1] * jump; // tangential + du[d] += sigma * B(p, d) * r_p; + u[d] += sigma * G(p, d) * w_p; + } + } + } + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD(side, y, 2) + { + real_t *u = (side == 0) ? u0 : u1; + real_t *du = (side == 0) ? du0 : du1; + MFEM_FOREACH_THREAD(d, x, D1D) + { + y(d, side, f) += u[d]; + dydn(d, side, f) += du[d]; + } + } + }); // mfem::forall +} + +template +static void PADGDiffusionApply3D(const int NF, const Array &b, + const Array &bt, + const Array &g, + const Array >, const real_t sigma, + const Vector &pa_data, const Vector &x_, + const Vector &dxdn_, Vector &y_, Vector &dydn_, + const int d1d = 0, const int q1d = 0) +{ + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + + auto B_ = Reshape(b.Read(), Q1D, D1D); + auto G_ = Reshape(g.Read(), Q1D, D1D); + + // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h) + auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF); + + auto x = Reshape(x_.Read(), D1D, D1D, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF); + auto dxdn = Reshape(dxdn_.Read(), D1D, D1D, 2, NF); + auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF); + + const int NBX = std::max(D1D, Q1D); + + mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void + { + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + + MFEM_SHARED real_t u0[max_Q1D][max_Q1D]; + MFEM_SHARED real_t u1[max_Q1D][max_Q1D]; + + MFEM_SHARED real_t du0[max_Q1D][max_Q1D]; + MFEM_SHARED real_t du1[max_Q1D][max_Q1D]; + + MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D]; + MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D]; + + MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D]; + MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D]; + + MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D]; + MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D]; + + MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D]; + + MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3]; + MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D]; + + // some buffers are reused multiple times, but for clarity have new names: + real_t(*Bj0)[max_Q1D] = Bu0; + real_t(*Bj1)[max_Q1D] = Bu1; + real_t(*Bjn0)[max_Q1D] = Bdu0; + real_t(*Bjn1)[max_Q1D] = Bdu1; + real_t(*Gj0)[max_Q1D] = Gu0; + real_t(*Gj1)[max_Q1D] = Gu1; + + DeviceMatrix B(BG, Q1D, D1D); + DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D); + + // copy face values to u0, u1 and copy normals to du0, du1 + MFEM_FOREACH_THREAD(side, z, 2) + { + real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1; + real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1; + + MFEM_FOREACH_THREAD(d2, x, D1D) + { + MFEM_FOREACH_THREAD(d1, y, D1D) + { + u[d2][d1] = x(d1, d2, side, + f); // copy transposed for better memory access + du[d2][d1] = dxdn(d1, d2, side, f); + } + } + + MFEM_FOREACH_THREAD(p1, x, Q1D) + { + MFEM_FOREACH_THREAD(p2, y, Q1D) + { + for (int l = 0; l < 3; ++l) + { + nJe[side][p2][p1][l] = pa(3 * side + l, p1, p2, f); + } + + if (side == 0) + { + kappa_Qh[p2][p1] = pa(6, p1, p2, f); + } + } + } + + if (side == 0) + { + MFEM_FOREACH_THREAD(p, x, Q1D) + { + MFEM_FOREACH_THREAD(d, y, D1D) + { + B(p, d) = B_(p, d); + G(p, d) = G_(p, d); + } + } + } + } + MFEM_SYNC_THREAD; + + // eval u and normal derivative @ quad points + MFEM_FOREACH_THREAD(side, z, 2) + { + real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1; + real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1; + real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1; + real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1; + real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1; + + MFEM_FOREACH_THREAD(p1, x, Q1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + real_t bu = 0.0; + real_t bdu = 0.0; + real_t gu = 0.0; + + for (int d1 = 0; d1 < D1D; ++d1) + { + const real_t b = B(p1, d1); + const real_t g = G(p1, d1); + + bu += b * u[d2][d1]; + bdu += b * du[d2][d1]; + gu += g * u[d2][d1]; + } + + Bu[p1][d2] = bu; + Bdu[p1][d2] = bdu; + Gu[p1][d2] = gu; + } + } + } + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD(side, z, 2) + { + real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1; + real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1; + real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1; + real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1; + real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1; + + MFEM_FOREACH_THREAD(p2, x, Q1D) + { + MFEM_FOREACH_THREAD(p1, y, Q1D) + { + const real_t *Je = nJe[side][p2][p1]; + + real_t bbu = 0.0; + real_t bgu = 0.0; + real_t gbu = 0.0; + real_t bbdu = 0.0; + + for (int d2 = 0; d2 < D1D; ++d2) + { + const real_t b = B(p2, d2); + const real_t g = G(p2, d2); + bbu += b * Bu[p1][d2]; + gbu += g * Bu[p1][d2]; + bgu += b * Gu[p1][d2]; + bbdu += b * Bdu[p1][d2]; + } + + u[p2][p1] = bbu; + // du <- Q du/dn * w * det(J) + du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu; + } + } + } + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD(side, z, 2) + { + real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1; + real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1; + real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1; + + MFEM_FOREACH_THREAD(d1, x, D1D) + { + MFEM_FOREACH_THREAD(p2, y, Q1D) + { + real_t bj = 0.0; + real_t bjn = 0.0; + real_t gj = 0.0; + real_t br = 0.0; + + for (int p1 = 0; p1 < Q1D; ++p1) + { + const real_t b = B(p1, d1); + const real_t g = G(p1, d1); + + const real_t *Je = nJe[side][p2][p1]; + + const real_t jump = u0[p2][p1] - u1[p2][p1]; + const real_t avg = du0[p2][p1] + du1[p2][p1]; + + // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] > + const real_t r = -avg + kappa_Qh[p2][p1] * jump; + + // bj, gj, bjn contribute to sigma term + bj += b * Je[0] * jump; + gj += g * Je[1] * jump; + bjn += b * Je[2] * jump; + + br += b * r; + } + + Bj[d1][p2] = sigma * bj; + Bjn[d1][p2] = sigma * bjn; + + // group br and gj together since we will multiply them both by B + // and then sum + const real_t sgn = (side == 0) ? 1.0 : -1.0; + Gj[d1][p2] = sgn * br + sigma * gj; + } + } + } + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD(side, z, 2) + { + real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1; + real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1; + real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1; + real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1; + real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1; + + MFEM_FOREACH_THREAD(d2, x, D1D) + { + MFEM_FOREACH_THREAD(d1, y, D1D) + { + real_t bbj = 0.0; + real_t gbj = 0.0; + real_t bgj = 0.0; + + for (int p2 = 0; p2 < Q1D; ++p2) + { + const real_t b = B(p2, d2); + const real_t g = G(p2, d2); + + bbj += b * Bj[d1][p2]; + bgj += b * Gj[d1][p2]; + gbj += g * Bjn[d1][p2]; + } + + du[d2][d1] = bbj; + u[d2][d1] = bgj + gbj; + } + } + } + MFEM_SYNC_THREAD; + + // map back to y and dydn + MFEM_FOREACH_THREAD(side, z, 2) + { + const real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1; + const real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1; + + MFEM_FOREACH_THREAD(d2, x, D1D) + { + MFEM_FOREACH_THREAD(d1, y, D1D) + { + y(d1, d2, side, f) += u[d2][d1]; + dydn(d1, d2, side, f) += du[d2][d1]; + } + } + } + }); +} + +} // namespace internal + +template +DGDiffusionIntegrator::ApplyKernelType +DGDiffusionIntegrator::ApplyPAKernels::Kernel() +{ + if constexpr (DIM == 2) + { + return internal::PADGDiffusionApply2D; + } + else if constexpr (DIM == 3) + { + return internal::PADGDiffusionApply3D; + } + MFEM_ABORT(""); +} +} // namespace mfem +/// \endcond DO_NOT_DOCUMENT +#endif diff --git a/fem/integ/bilininteg_dgdiffusion_pa.cpp b/fem/integ/bilininteg_dgdiffusion_pa.cpp index fc0cd3a443..c4488f6a71 100644 --- a/fem/integ/bilininteg_dgdiffusion_pa.cpp +++ b/fem/integ/bilininteg_dgdiffusion_pa.cpp @@ -11,42 +11,39 @@ #include "../../general/forall.hpp" #include "../../mesh/face_nbr_geom.hpp" +#include "../fe/face_map_utils.hpp" #include "../gridfunc.hpp" #include "../qfunction.hpp" -#include "../fe/face_map_utils.hpp" -using namespace std; +#include "bilininteg_dgdiffusion_kernels.hpp" namespace mfem { -static void PADGDiffusionSetup2D(const int Q1D, - const int NE, - const int NF, +static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF, const Array &w, const GeometricFactors &el_geom, const FaceGeometricFactors &face_geom, const FaceNeighborGeometricFactors *nbr_geom, - const Vector &q, - const real_t sigma, - const real_t kappa, - Vector &pa_data, + const Vector &q, const real_t sigma, + const real_t kappa, Vector &pa_data, const Array &face_info_) { const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, 2, 2, NE); const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, NE); const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0; - const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, - Q1D, Q1D, 2, 2, n_nbr); - const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, - Q1D, Q1D, n_nbr); + const auto J_shared = + Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D, Q1D, 2, 2, n_nbr); + const auto detJ_shared = + Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, n_nbr); const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, NF); const auto n = Reshape(face_geom.normal.Read(), Q1D, 2, NF); const bool const_q = (q.Size() == 1); - const auto Q = const_q ? Reshape(q.Read(), 1,1) : Reshape(q.Read(), Q1D,NF); + const auto Q = + const_q ? Reshape(q.Read(), 1, 1) : Reshape(q.Read(), Q1D, NF); const auto W = w.Read(); @@ -56,7 +53,7 @@ static void PADGDiffusionSetup2D(const int Q1D, // (q, 1/h, J0_0, J0_1, J1_0, J1_1) auto pa = Reshape(pa_data.Write(), 6, Q1D, NF); - mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) -> void + mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) -> void { const int normal_dir[] = {face_info(0, f), face_info(1, f)}; const int fid[] = {face_info(4, f), face_info(5, f)}; @@ -74,7 +71,7 @@ static void PADGDiffusionSetup2D(const int Q1D, for (int p = 0; p < Q1D; ++p) { - const real_t Qp = const_q ? Q(0,0) : Q(p, f); + const real_t Qp = const_q ? Q(0, 0) : Q(p, f); pa(0, p, f) = kappa * Qp * W[p] * detJf(p, f); real_t hi = 0.0; @@ -85,17 +82,19 @@ static void PADGDiffusionSetup2D(const int Q1D, // Always opposite direction in "native" ordering // Need to multiply the native=>lex0 with native=>lex1 and negate - const int sgn = (side == 1) ? -1*sgn0*sgn1 : 1; + const int sgn = (side == 1) ? -1 * sgn0 * sgn1 : 1; const int e = el[side]; const auto &J = (side == 1 && shared) ? J_shared : J_loc; const auto &detJ = (side == 1 && shared) ? detJ_shared : detJe_loc; real_t nJi[2]; - nJi[0] = n(p,0,f)*J(i,j, 1,1, e) - n(p,1,f)*J(i,j,0,1,e); - nJi[1] = -n(p,0,f)*J(i,j,1,0, e) + n(p,1,f)*J(i,j,0,0,e); + nJi[0] = + n(p, 0, f) * J(i, j, 1, 1, e) - n(p, 1, f) * J(i, j, 0, 1, e); + nJi[1] = + -n(p, 0, f) * J(i, j, 1, 0, e) + n(p, 1, f) * J(i, j, 0, 0, e); - const real_t dJe = detJ(i,j,e); + const real_t dJe = detJ(i, j, e); const real_t dJf = detJf(p, f); const real_t w = factor * Qp * W[p] * dJf / dJe; @@ -104,9 +103,9 @@ static void PADGDiffusionSetup2D(const int Q1D, const int ti = 1 - ni; // Normal - pa(2 + 2*side + 0, p, f) = w * nJi[ni]; + pa(2 + 2 * side + 0, p, f) = w * nJi[ni]; // Tangential - pa(2 + 2*side + 1, p, f) = sgn * w * nJi[ti]; + pa(2 + 2 * side + 1, p, f) = sgn * w * nJi[ti]; hi += factor * dJf / dJe; } @@ -122,47 +121,43 @@ static void PADGDiffusionSetup2D(const int Q1D, }); } -static void PADGDiffusionSetup3D(const int Q1D, - const int NE, - const int NF, +static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF, const Array &w, const GeometricFactors &el_geom, const FaceGeometricFactors &face_geom, const FaceNeighborGeometricFactors *nbr_geom, - const Vector &q, - const real_t sigma, - const real_t kappa, - Vector &pa_data, + const Vector &q, const real_t sigma, + const real_t kappa, Vector &pa_data, const Array &face_info_) { const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, Q1D, 3, 3, NE); const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, Q1D, NE); const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0; - const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, - Q1D, Q1D, Q1D, 3, 3, n_nbr); - const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, - Q1D, Q1D, Q1D, n_nbr); + const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D, + Q1D, Q1D, 3, 3, n_nbr); + const auto detJ_shared = + Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, Q1D, n_nbr); const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, Q1D, NF); const auto n = Reshape(face_geom.normal.Read(), Q1D, Q1D, 3, NF); const bool const_q = (q.Size() == 1); - const auto Q = const_q ? Reshape(q.Read(), 1, 1, 1) - : Reshape(q.Read(), Q1D, Q1D, NF); + const auto Q = + const_q ? Reshape(q.Read(), 1, 1, 1) : Reshape(q.Read(), Q1D, Q1D, NF); const auto W = Reshape(w.Read(), Q1D, Q1D); // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation) const auto face_info = Reshape(face_info_.Read(), 6, 2, NF); - constexpr int _el_ = 3; // offset in face_info for element index + constexpr int _el_ = 3; // offset in face_info for element index constexpr int _fid_ = 4; // offset in face_info for local face id - constexpr int _or_ = 5; // offset in face_info for orientation + constexpr int _or_ = 5; // offset in face_info for orientation // (J00, J01, J02, J10, J11, J12, q/h) const auto pa = Reshape(pa_data.Write(), 7, Q1D, Q1D, NF); - mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int f) -> void + mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int f) -> void { MFEM_SHARED int perm[2][3]; MFEM_SHARED int el[2]; @@ -172,10 +167,7 @@ static void PADGDiffusionSetup3D(const int Q1D, MFEM_FOREACH_THREAD(side, x, 2) { - MFEM_FOREACH_THREAD(i, y, 3) - { - perm[side][i] = face_info(i, side, f); - } + MFEM_FOREACH_THREAD(i, y, 3) { perm[side][i] = face_info(i, side, f); } if (MFEM_THREAD_ID(y) == 0) { @@ -200,16 +192,16 @@ static void PADGDiffusionSetup3D(const int Q1D, { MFEM_FOREACH_THREAD(p2, y, Q1D) { - const real_t Qp = const_q ? Q(0,0,0) : Q(p1, p2, f); - const real_t dJf = detJf(p1,p2,f); + const real_t Qp = const_q ? Q(0, 0, 0) : Q(p1, p2, f); + const real_t dJf = detJf(p1, p2, f); real_t hi = 0.0; for (int side = 0; side < nsides; ++side) { int i, j, k; - internal::FaceIdxToVolIdx3D( - p1 + Q1D*p2, Q1D, fid[0], fid[1], side, ortn[1], i, j, k); + internal::FaceIdxToVolIdx3D(p1 + Q1D * p2, Q1D, fid[0], fid[1], + side, ortn[1], i, j, k); const int e = el[side]; const auto &J = shared[side] ? J_shared : J_loc; @@ -217,27 +209,45 @@ static void PADGDiffusionSetup3D(const int Q1D, // *INDENT-OFF* real_t nJi[3]; - nJi[0] = ( -J(i,j,k, 1,2, e)*J(i,j,k, 2,1, e) + J(i,j,k, 1,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f) - + ( J(i,j,k, 0,2, e)*J(i,j,k, 2,1, e) - J(i,j,k, 0,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f) - + (-J(i,j,k, 0,2, e)*J(i,j,k, 1,1, e) + J(i,j,k, 0,1, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f); + nJi[0] = (-J(i, j, k, 1, 2, e) * J(i, j, k, 2, 1, e) + + J(i, j, k, 1, 1, e) * J(i, j, k, 2, 2, e)) * + n(p1, p2, 0, f) + + (J(i, j, k, 0, 2, e) * J(i, j, k, 2, 1, e) - + J(i, j, k, 0, 1, e) * J(i, j, k, 2, 2, e)) * + n(p1, p2, 1, f) + + (-J(i, j, k, 0, 2, e) * J(i, j, k, 1, 1, e) + + J(i, j, k, 0, 1, e) * J(i, j, k, 1, 2, e)) * + n(p1, p2, 2, f); - nJi[1] = ( J(i,j,k, 1,2, e)*J(i,j,k, 2,0, e) - J(i,j,k, 1,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f) - + (-J(i,j,k, 0,2, e)*J(i,j,k, 2,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f) - + ( J(i,j,k, 0,2, e)*J(i,j,k, 1,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f); + nJi[1] = (J(i, j, k, 1, 2, e) * J(i, j, k, 2, 0, e) - + J(i, j, k, 1, 0, e) * J(i, j, k, 2, 2, e)) * + n(p1, p2, 0, f) + + (-J(i, j, k, 0, 2, e) * J(i, j, k, 2, 0, e) + + J(i, j, k, 0, 0, e) * J(i, j, k, 2, 2, e)) * + n(p1, p2, 1, f) + + (J(i, j, k, 0, 2, e) * J(i, j, k, 1, 0, e) - + J(i, j, k, 0, 0, e) * J(i, j, k, 1, 2, e)) * + n(p1, p2, 2, f); - nJi[2] = ( -J(i,j,k, 1,1, e)*J(i,j,k, 2,0, e) + J(i,j,k, 1,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 0, f) - + ( J(i,j,k, 0,1, e)*J(i,j,k, 2,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 1, f) - + (-J(i,j,k, 0,1, e)*J(i,j,k, 1,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 1,1, e)) * n(p1,p2, 2, f); + nJi[2] = (-J(i, j, k, 1, 1, e) * J(i, j, k, 2, 0, e) + + J(i, j, k, 1, 0, e) * J(i, j, k, 2, 1, e)) * + n(p1, p2, 0, f) + + (J(i, j, k, 0, 1, e) * J(i, j, k, 2, 0, e) - + J(i, j, k, 0, 0, e) * J(i, j, k, 2, 1, e)) * + n(p1, p2, 1, f) + + (-J(i, j, k, 0, 1, e) * J(i, j, k, 1, 0, e) + + J(i, j, k, 0, 0, e) * J(i, j, k, 1, 1, e)) * + n(p1, p2, 2, f); // *INDENT-ON* - const real_t dJe = detJe(i,j,k,e); + const real_t dJe = detJe(i, j, k, e); const real_t val = factor * Qp * W(p1, p2) * dJf / dJe; for (int d = 0; d < 3; ++d) { const int idx = std::abs(perm[side][d]) - 1; const int sgn = (perm[side][d] < 0) ? -1 : 1; - pa(3*side + d, p1, p2, f) = sgn * val * nJi[idx]; + pa(3 * side + d, p1, p2, f) = sgn * val * nJi[idx]; } hi += factor * dJf / dJe; @@ -257,7 +267,8 @@ static void PADGDiffusionSetup3D(const int Q1D, } static void PADGDiffusionSetupFaceInfo2D(const int nf, const Mesh &mesh, - const FaceType type, Array &face_info_) + const FaceType type, + Array &face_info_) { const int ne = mesh.GetNE(); @@ -326,8 +337,7 @@ inline void FaceNormalPermutation(int perm[3], const int face_id) // Assigns to perm the permutation as in FaceNormalPermutation for the second // element on the face but signed to indicate the sign of the normal derivative. -inline void SignedFaceNormalPermutation(int perm[3], - const int face_id1, +inline void SignedFaceNormalPermutation(int perm[3], const int face_id1, const int face_id2, const int orientation) { @@ -386,17 +396,19 @@ inline void SignedFaceNormalPermutation(int perm[3], } static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh, - const FaceType type, Array &face_info_) + const FaceType type, + Array &face_info_) { const int ne = mesh.GetNE(); int fidx = 0; - // face_info array has 12 entries per face, 6 for each of the adjacent elements: - // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation) + // face_info array has 12 entries per face, 6 for each of the adjacent + // elements: (perm[0], perm[1], perm[2], element_index, local_face_id, + // orientation) face_info_.SetSize(nf * 12); - constexpr int _e_ = 3; // offset for element index + constexpr int _e_ = 3; // offset for element index constexpr int _fid_ = 4; // offset for local face id - constexpr int _or_ = 5; // offset for orientation + constexpr int _or_ = 5; // offset for orientation auto face_info = Reshape(face_info_.HostWrite(), 6, 2, nf); for (int f = 0; f < mesh.GetNumFaces(); ++f) @@ -408,9 +420,9 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh, const int fid0 = f_info.element[0].local_face_id; const int or0 = f_info.element[0].orientation; - face_info( _e_, 0, fidx) = f_info.element[0].index; + face_info(_e_, 0, fidx) = f_info.element[0].index; face_info(_fid_, 0, fidx) = fid0; - face_info( _or_, 0, fidx) = or0; + face_info(_or_, 0, fidx) = or0; FaceNormalPermutation(&face_info(0, 0, fidx), fid0); @@ -421,16 +433,17 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh, if (f_info.IsShared()) { - face_info( _e_, 1, fidx) = ne + f_info.element[1].index; + face_info(_e_, 1, fidx) = ne + f_info.element[1].index; } else { - face_info( _e_, 1, fidx) = f_info.element[1].index; + face_info(_e_, 1, fidx) = f_info.element[1].index; } face_info(_fid_, 1, fidx) = fid1; - face_info( _or_, 1, fidx) = or1; + face_info(_or_, 1, fidx) = or1; - SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1, or1); + SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1, + or1); } else { @@ -448,8 +461,8 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh, void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type) { - const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ? - Device::GetDeviceMemoryType() : pa_mt; + const MemoryType mt = + (pa_mt == MemoryType::DEFAULT) ? Device::GetDeviceMemoryType() : pa_mt; const int ne = fes.GetNE(); nf = fes.GetNFbyType(type); @@ -458,16 +471,17 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes, Mesh &mesh = *fes.GetMesh(); const Geometry::Type face_geom_type = mesh.GetTypicalFaceGeometry(); const FiniteElement &el = *fes.GetTypicalTraceElement(); - const int ir_order = IntRule ? IntRule->GetOrder() + const int ir_order = IntRule + ? IntRule->GetOrder() : GetRule(el.GetOrder(), face_geom_type).GetOrder(); const IntegrationRule &ir = irs.Get(face_geom_type, ir_order); dim = mesh.Dimension(); - const int q1d = (ir.GetOrder() + 3)/2; - MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0/(dim - 1)), ""); + const int q1d = (ir.GetOrder() + 3) / 2; + MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0 / (dim - 1)), ""); const auto vol_ir = irs.Get(mesh.GetTypicalElementGeometry(), ir_order); - const auto geom_flags = GeometricFactors::JACOBIANS | - GeometricFactors::DETERMINANTS; + const auto geom_flags = + GeometricFactors::JACOBIANS | GeometricFactors::DETERMINANTS; const auto el_geom = mesh.GetGeometricFactors(vol_ir, geom_flags, mt); std::unique_ptr nbr_geom; @@ -476,8 +490,8 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes, nbr_geom.reset(new FaceNeighborGeometricFactors(*el_geom)); } - const auto face_geom_flags = FaceGeometricFactors::DETERMINANTS | - FaceGeometricFactors::NORMALS; + const auto face_geom_flags = + FaceGeometricFactors::DETERMINANTS | FaceGeometricFactors::NORMALS; auto face_geom = mesh.GetFaceGeometricFactors(ir, face_geom_flags, type, mt); maps = &el.GetDofToQuad(ir, DofToQuad::TENSOR); dofs1D = maps->ndof; @@ -489,9 +503,18 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes, // Evaluate the coefficient at the face quadrature points. FaceQuadratureSpace fqs(mesh, ir, type); CoefficientVector q(fqs, CoefficientStorage::COMPRESSED); - if (Q) { q.Project(*Q); } - else if (MQ) { MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */ } - else { q.SetConstant(1.0); } + if (Q) + { + q.Project(*Q); + } + else if (MQ) + { + MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */ + } + else + { + q.SetConstant(1.0); + } Array face_info; if (dim == 1) @@ -501,14 +524,16 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes, else if (dim == 2) { PADGDiffusionSetupFaceInfo2D(nf, mesh, type, face_info); - PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom, - nbr_geom.get(), q, sigma, kappa, pa_data, face_info); + PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom, + *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data, + face_info); } else if (dim == 3) { PADGDiffusionSetupFaceInfo3D(nf, mesh, type, face_info); - PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom, - nbr_geom.get(), q, sigma, kappa, pa_data, face_info); + PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom, + *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data, + face_info); } } @@ -524,529 +549,76 @@ void DGDiffusionIntegrator::AssemblePABoundaryFaces( SetupPA(fes, FaceType::Boundary); } -template static -void PADGDiffusionApply2D(const int NF, - const Array &b, - const Array &bt, - const Array& g, - const Array& gt, - const real_t sigma, - const Vector &pa_data, - const Vector &x_, - const Vector &dxdn_, - Vector &y_, - Vector &dydn_, - const int d1d = 0, - const int q1d = 0) +void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives(const Vector &x, + const Vector &dxdn, + Vector &y, + Vector &dydn) const { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - - auto B_ = Reshape(b.Read(), Q1D, D1D); - auto G_ = Reshape(g.Read(), Q1D, D1D); - - auto pa = Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11) - - auto x = Reshape(x_.Read(), D1D, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, 2, NF); - auto dxdn = Reshape(dxdn_.Read(), D1D, 2, NF); - auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF); - - const int NBX = std::max(D1D, Q1D); - - mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void - { - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - - MFEM_SHARED real_t u0[max_D1D]; - MFEM_SHARED real_t u1[max_D1D]; - MFEM_SHARED real_t du0[max_D1D]; - MFEM_SHARED real_t du1[max_D1D]; - - MFEM_SHARED real_t Bu0[max_Q1D]; - MFEM_SHARED real_t Bu1[max_Q1D]; - MFEM_SHARED real_t Bdu0[max_Q1D]; - MFEM_SHARED real_t Bdu1[max_Q1D]; - - MFEM_SHARED real_t r[max_Q1D]; - - MFEM_SHARED real_t BG[2*max_D1D*max_Q1D]; - DeviceMatrix B(BG, Q1D, D1D); - DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D); - - if (MFEM_THREAD_ID(y) == 0) - { - MFEM_FOREACH_THREAD(p,x,Q1D) - { - for (int d = 0; d < D1D; ++d) - { - B(p,d) = B_(p,d); - G(p,d) = G_(p,d); - } - } - } - MFEM_SYNC_THREAD; - - // copy edge values to u0, u1 and copy edge normals to du0, du1 - MFEM_FOREACH_THREAD(side,y,2) - { - real_t *u = (side == 0) ? u0 : u1; - real_t *du = (side == 0) ? du0 : du1; - MFEM_FOREACH_THREAD(d,x,D1D) - { - u[d] = x(d, side, f); - du[d] = dxdn(d, side, f); - } - } - MFEM_SYNC_THREAD; - - // eval @ quad points - MFEM_FOREACH_THREAD(side,y,2) - { - real_t *u = (side == 0) ? u0 : u1; - real_t *du = (side == 0) ? du0 : du1; - real_t *Bu = (side == 0) ? Bu0 : Bu1; - real_t *Bdu = (side == 0) ? Bdu0 : Bdu1; - - MFEM_FOREACH_THREAD(p,x,Q1D) - { - const real_t Je_side[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)}; - - Bu[p] = 0.0; - Bdu[p] = 0.0; - - for (int d = 0; d < D1D; ++d) - { - const real_t b = B(p,d); - const real_t g = G(p,d); - - Bu[p] += b*u[d]; - Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d]; - } - } - } - MFEM_SYNC_THREAD; - - // term - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >: - if (MFEM_THREAD_ID(y) == 0) - { - MFEM_FOREACH_THREAD(p,x,Q1D) - { - const real_t q = pa(0, p, f); - const real_t hi = pa(1, p, f); - const real_t jump = Bu0[p] - Bu1[p]; - const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J) - r[p] = -avg + hi * q * jump; - } - } - MFEM_SYNC_THREAD; - - MFEM_FOREACH_THREAD(d,x,D1D) - { - real_t Br = 0.0; - - for (int p = 0; p < Q1D; ++p) - { - Br += B(p, d) * r[p]; - } - - u0[d] = Br; // overwrite u0, u1 - u1[d] = -Br; - } // for d - MFEM_SYNC_THREAD; - - - MFEM_FOREACH_THREAD(side,y,2) - { - real_t *du = (side == 0) ? du0 : du1; - MFEM_FOREACH_THREAD(d,x,D1D) - { - du[d] = 0.0; - } - } - MFEM_SYNC_THREAD; - - // term sigma * < [u], {Q dv/dn} > - MFEM_FOREACH_THREAD(side,y,2) - { - real_t * const du = (side == 0) ? du0 : du1; - real_t * const u = (side == 0) ? u0 : u1; - - MFEM_FOREACH_THREAD(d,x,D1D) - { - for (int p = 0; p < Q1D; ++p) - { - const real_t Je[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)}; - const real_t jump = Bu0[p] - Bu1[p]; - const real_t r_p = Je[0] * jump; // normal - const real_t w_p = Je[1] * jump; // tangential - du[d] += sigma * B(p, d) * r_p; - u[d] += sigma * G(p, d) * w_p; - } - } - } - MFEM_SYNC_THREAD; - - MFEM_FOREACH_THREAD(side,y,2) - { - real_t *u = (side == 0) ? u0 : u1; - real_t *du = (side == 0) ? du0 : du1; - MFEM_FOREACH_THREAD(d,x,D1D) - { - y(d, side, f) += u[d]; - dydn(d, side, f) += du[d]; - } - } - }); // mfem::forall + ApplyPAKernels::Run(dim, dofs1D, quad1D, nf, maps->B, maps->Bt, maps->G, + maps->Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D, + quad1D); } -template -static void PADGDiffusionApply3D(const int NF, - const Array& b, - const Array& bt, - const Array& g, - const Array& gt, - const real_t sigma, - const Vector& pa_data, - const Vector& x_, - const Vector& dxdn_, - Vector& y_, - Vector& dydn_, - const int d1d = 0, - const int q1d = 0) +DGDiffusionIntegrator::DGDiffusionIntegrator(const real_t s, const real_t k) + : sigma(s), kappa(k) { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - - auto B_ = Reshape(b.Read(), Q1D, D1D); - auto G_ = Reshape(g.Read(), Q1D, D1D); - - // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h) - auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF); - - auto x = Reshape(x_.Read(), D1D, D1D, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF); - auto dxdn = Reshape(dxdn_.Read(), D1D, D1D, 2, NF); - auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF); - - const int NBX = std::max(D1D, Q1D); - - mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void - { - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - - MFEM_SHARED real_t u0[max_Q1D][max_Q1D]; - MFEM_SHARED real_t u1[max_Q1D][max_Q1D]; - - MFEM_SHARED real_t du0[max_Q1D][max_Q1D]; - MFEM_SHARED real_t du1[max_Q1D][max_Q1D]; - - MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D]; - MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D]; - - MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D]; - MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D]; - - MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D]; - MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D]; - - MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D]; - - MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3]; - MFEM_SHARED real_t BG[2*max_D1D*max_Q1D]; - - // some buffers are reused multiple times, but for clarity have new names: - real_t (*Bj0)[max_Q1D] = Bu0; - real_t (*Bj1)[max_Q1D] = Bu1; - real_t (*Bjn0)[max_Q1D] = Bdu0; - real_t (*Bjn1)[max_Q1D] = Bdu1; - real_t (*Gj0)[max_Q1D] = Gu0; - real_t (*Gj1)[max_Q1D] = Gu1; - - DeviceMatrix B(BG, Q1D, D1D); - DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D); - - // copy face values to u0, u1 and copy normals to du0, du1 - MFEM_FOREACH_THREAD(side, z, 2) - { - real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1; - real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1; - - MFEM_FOREACH_THREAD(d2, x, D1D) - { - MFEM_FOREACH_THREAD(d1, y, D1D) - { - u[d2][d1] = x(d1, d2, side, f); // copy transposed for better memory access - du[d2][d1] = dxdn(d1, d2, side, f); - } - } - - MFEM_FOREACH_THREAD(p1, x, Q1D) - { - MFEM_FOREACH_THREAD(p2, y, Q1D) - { - for (int l=0; l < 3; ++l) - { - nJe[side][p2][p1][l] = pa(3*side + l, p1, p2, f); - } - - if (side == 0) - { - kappa_Qh[p2][p1] = pa(6, p1, p2, f); - } - } - } - - if (side == 0) - { - MFEM_FOREACH_THREAD(p, x, Q1D) - { - MFEM_FOREACH_THREAD(d, y, D1D) - { - B(p, d) = B_(p, d); - G(p, d) = G_(p, d); - } - } - } - } - MFEM_SYNC_THREAD; - - // eval u and normal derivative @ quad points - MFEM_FOREACH_THREAD(side, z, 2) - { - real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1; - real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1; - real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1; - real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1; - real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1; - - MFEM_FOREACH_THREAD(p1, x, Q1D) - { - MFEM_FOREACH_THREAD(d2, y, D1D) - { - real_t bu = 0.0; - real_t bdu = 0.0; - real_t gu = 0.0; - - for (int d1=0; d1 < D1D; ++d1) - { - const real_t b = B(p1, d1); - const real_t g = G(p1, d1); - - bu += b * u[d2][d1]; - bdu += b * du[d2][d1]; - gu += g * u[d2][d1]; - } - - Bu[p1][d2] = bu; - Bdu[p1][d2] = bdu; - Gu[p1][d2] = gu; - } - } - } - MFEM_SYNC_THREAD; - - MFEM_FOREACH_THREAD(side, z, 2) - { - real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1; - real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1; - real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1; - real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1; - real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1; - - MFEM_FOREACH_THREAD(p2, x, Q1D) - { - MFEM_FOREACH_THREAD(p1, y, Q1D) - { - const real_t * Je = nJe[side][p2][p1]; - - real_t bbu = 0.0; - real_t bgu = 0.0; - real_t gbu = 0.0; - real_t bbdu = 0.0; - - for (int d2 = 0; d2 < D1D; ++d2) - { - const real_t b = B(p2, d2); - const real_t g = G(p2, d2); - bbu += b * Bu[p1][d2]; - gbu += g * Bu[p1][d2]; - bgu += b * Gu[p1][d2]; - bbdu += b * Bdu[p1][d2]; - } - - u[p2][p1] = bbu; - // du <- Q du/dn * w * det(J) - du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu; - } - } - } - MFEM_SYNC_THREAD; - - MFEM_FOREACH_THREAD(side, z, 2) - { - real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1; - real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1; - real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1; - - MFEM_FOREACH_THREAD(d1, x, D1D) - { - MFEM_FOREACH_THREAD(p2, y, Q1D) - { - real_t bj = 0.0; - real_t bjn = 0.0; - real_t gj = 0.0; - real_t br = 0.0; - - for (int p1 = 0; p1 < Q1D; ++p1) - { - const real_t b = B(p1, d1); - const real_t g = G(p1, d1); - - const real_t * Je = nJe[side][p2][p1]; - - const real_t jump = u0[p2][p1] - u1[p2][p1]; - const real_t avg = du0[p2][p1] + du1[p2][p1]; - - // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] > - const real_t r = -avg + kappa_Qh[p2][p1] * jump; - - // bj, gj, bjn contribute to sigma term - bj += b * Je[0] * jump; - gj += g * Je[1] * jump; - bjn += b * Je[2] * jump; - - br += b * r; - } - - Bj[d1][p2] = sigma * bj; - Bjn[d1][p2] = sigma * bjn; - - // group br and gj together since we will multiply them both by B - // and then sum - const real_t sgn = (side == 0) ? 1.0 : -1.0; - Gj[d1][p2] = sgn * br + sigma * gj; - } - } - } - MFEM_SYNC_THREAD; - - MFEM_FOREACH_THREAD(side, z, 2) - { - real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1; - real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1; - real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1; - real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1; - real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1; - - MFEM_FOREACH_THREAD(d2, x, D1D) - { - MFEM_FOREACH_THREAD(d1, y, D1D) - { - real_t bbj = 0.0; - real_t gbj = 0.0; - real_t bgj = 0.0; - - for (int p2 = 0; p2 < Q1D; ++p2) - { - const real_t b = B(p2, d2); - const real_t g = G(p2, d2); - - bbj += b * Bj[d1][p2]; - bgj += b * Gj[d1][p2]; - gbj += g * Bjn[d1][p2]; - } - - du[d2][d1] = bbj; - u[d2][d1] = bgj + gbj; - } - } - } - MFEM_SYNC_THREAD; - - // map back to y and dydn - MFEM_FOREACH_THREAD(side, z, 2) - { - const real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1; - const real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1; - - MFEM_FOREACH_THREAD(d2, x, D1D) - { - MFEM_FOREACH_THREAD(d1, y, D1D) - { - y(d1, d2, side, f) += u[d2][d1]; - dydn(d1, d2, side, f) += du[d2][d1]; - } - } - } - }); + static Kernels kernels; } -static void PADGDiffusionApply(const int dim, - const int D1D, - const int Q1D, - const int NF, - const Array &B, - const Array &Bt, - const Array &G, - const Array &Gt, - const real_t sigma, - const Vector &pa_data, - const Vector &x, - const Vector &dxdn, - Vector &y, - Vector &dydn) +DGDiffusionIntegrator::DGDiffusionIntegrator(Coefficient &q, const real_t s, + const real_t k) + : DGDiffusionIntegrator(s, k) +{ + Q = &q; +} + +DGDiffusionIntegrator::DGDiffusionIntegrator(MatrixCoefficient &q, + const real_t s, const real_t k) + : DGDiffusionIntegrator(s, k) +{ + MQ = &q; +} + +/// \cond DO_NOT_DOCUMENT + +DGDiffusionIntegrator::ApplyKernelType +DGDiffusionIntegrator::ApplyPAKernels::Fallback(int dim, int, int) { if (dim == 2) { - auto kernel = PADGDiffusionApply2D<0,0>; - switch ((D1D << 4 ) | Q1D) - { - case 0x23: kernel = PADGDiffusionApply2D<2,3>; break; - case 0x34: kernel = PADGDiffusionApply2D<3,4>; break; - case 0x45: kernel = PADGDiffusionApply2D<4,5>; break; - case 0x56: kernel = PADGDiffusionApply2D<5,6>; break; - case 0x67: kernel = PADGDiffusionApply2D<6,7>; break; - case 0x78: kernel = PADGDiffusionApply2D<7,8>; break; - case 0x89: kernel = PADGDiffusionApply2D<8,9>; break; - case 0x9A: kernel = PADGDiffusionApply2D<9,10>; break; - } - kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D); + return internal::PADGDiffusionApply2D; } else if (dim == 3) { - auto kernel = PADGDiffusionApply3D<0,0>; - switch ((D1D << 4) | Q1D) - { - case 0x24: kernel = PADGDiffusionApply3D<2,4>; break; - case 0x35: kernel = PADGDiffusionApply3D<3,5>; break; - case 0x46: kernel = PADGDiffusionApply3D<4,6>; break; - case 0x57: kernel = PADGDiffusionApply3D<5,7>; break; - case 0x68: kernel = PADGDiffusionApply3D<6,8>; break; - case 0x79: kernel = PADGDiffusionApply3D<7,9>; break; - case 0x8A: kernel = PADGDiffusionApply3D<8,10>; break; - case 0x9B: kernel = PADGDiffusionApply3D<9,11>; break; - } - kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D); + return internal::PADGDiffusionApply3D; } else { - MFEM_ABORT("Unsupported dimension"); + MFEM_ABORT(""); } } -void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives( - const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const +DGDiffusionIntegrator::Kernels::Kernels() { - PADGDiffusionApply(dim, dofs1D, quad1D, nf, - maps->B, maps->Bt, maps->G, maps->Gt, - sigma, pa_data, x, dxdn, y, dydn); + DGDiffusionIntegrator::AddSpecialization<2, 2, 3>(); + DGDiffusionIntegrator::AddSpecialization<2, 3, 4>(); + DGDiffusionIntegrator::AddSpecialization<2, 4, 5>(); + DGDiffusionIntegrator::AddSpecialization<2, 5, 6>(); + DGDiffusionIntegrator::AddSpecialization<2, 6, 7>(); + DGDiffusionIntegrator::AddSpecialization<2, 7, 8>(); + DGDiffusionIntegrator::AddSpecialization<2, 8, 9>(); + DGDiffusionIntegrator::AddSpecialization<2, 9, 10>(); + + DGDiffusionIntegrator::AddSpecialization<3, 2, 4>(); + DGDiffusionIntegrator::AddSpecialization<3, 3, 5>(); + DGDiffusionIntegrator::AddSpecialization<3, 4, 6>(); + DGDiffusionIntegrator::AddSpecialization<3, 5, 7>(); + DGDiffusionIntegrator::AddSpecialization<3, 6, 8>(); + DGDiffusionIntegrator::AddSpecialization<3, 7, 9>(); + DGDiffusionIntegrator::AddSpecialization<3, 8, 10>(); + DGDiffusionIntegrator::AddSpecialization<3, 9, 11>(); } +/// \endcond DO_NOT_DOCUMENT + } // namespace mfem diff --git a/fem/integ/bilininteg_dgtrace_kernels.hpp b/fem/integ/bilininteg_dgtrace_kernels.hpp new file mode 100644 index 0000000000..6606be0770 --- /dev/null +++ b/fem/integ/bilininteg_dgtrace_kernels.hpp @@ -0,0 +1,793 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef BILININTEG_DGTRACE_KERNELS_HPP +#define BILININTEG_DGTRACE_KERNELS_HPP + +#include "../../general/forall.hpp" +#include "../bilininteg.hpp" +#include "../gridfunc.hpp" +#include "../qfunction.hpp" +#include "../restriction.hpp" + +/// \cond DO_NOT_DOCUMENT +namespace mfem +{ + +namespace internal +{ + +// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein +template +static void PADGTraceApply2D(const int NF, const Array &b, + const Array &bt, const Vector &op_, + const Vector &x_, Vector &y_, const int d1d = 0, + const int q1d = 0) +{ + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, 2, 2, NF); + auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF); + + mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) + { + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + real_t u0[max_D1D][VDIM]; + real_t u1[max_D1D][VDIM]; + for (int d = 0; d < D1D; d++) + { + for (int c = 0; c < VDIM; c++) + { + u0[d][c] = x(d, c, 0, f); + u1[d][c] = x(d, c, 1, f); + } + } + real_t Bu0[max_Q1D][VDIM]; + real_t Bu1[max_Q1D][VDIM]; + for (int q = 0; q < Q1D; ++q) + { + for (int c = 0; c < VDIM; c++) + { + Bu0[q][c] = 0.0; + Bu1[q][c] = 0.0; + } + for (int d = 0; d < D1D; ++d) + { + const real_t b = B(q, d); + for (int c = 0; c < VDIM; c++) + { + Bu0[q][c] += b * u0[d][c]; + Bu1[q][c] += b * u1[d][c]; + } + } + } + real_t DBu[max_Q1D][VDIM]; + for (int q = 0; q < Q1D; ++q) + { + for (int c = 0; c < VDIM; c++) + { + DBu[q][c] = op(q, 0, 0, f) * Bu0[q][c] + op(q, 1, 0, f) * Bu1[q][c]; + } + } + real_t BDBu[max_D1D][VDIM]; + for (int d = 0; d < D1D; ++d) + { + for (int c = 0; c < VDIM; c++) + { + BDBu[d][c] = 0.0; + } + for (int q = 0; q < Q1D; ++q) + { + const real_t b = Bt(d, q); + for (int c = 0; c < VDIM; c++) + { + BDBu[d][c] += b * DBu[q][c]; + } + } + for (int c = 0; c < VDIM; c++) + { + y(d, c, 0, f) += BDBu[d][c]; + y(d, c, 1, f) += -BDBu[d][c]; + } + } + }); +} + +// PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein +template +static void PADGTraceApply3D(const int NF, const Array &b, + const Array &bt, const Vector &op_, + const Vector &x_, Vector &y_, const int d1d = 0, + const int q1d = 0) +{ + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); + auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF); + + mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) + { + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + real_t u0[max_D1D][max_D1D][VDIM]; + real_t u1[max_D1D][max_D1D][VDIM]; + for (int d1 = 0; d1 < D1D; d1++) + { + for (int d2 = 0; d2 < D1D; d2++) + { + for (int c = 0; c < VDIM; c++) + { + u0[d1][d2][c] = x(d1, d2, c, 0, f); + u1[d1][d2][c] = x(d1, d2, c, 1, f); + } + } + } + real_t Bu0[max_Q1D][max_D1D][VDIM]; + real_t Bu1[max_Q1D][max_D1D][VDIM]; + for (int q = 0; q < Q1D; ++q) + { + for (int d2 = 0; d2 < D1D; d2++) + { + for (int c = 0; c < VDIM; c++) + { + Bu0[q][d2][c] = 0.0; + Bu1[q][d2][c] = 0.0; + } + for (int d1 = 0; d1 < D1D; ++d1) + { + const real_t b = B(q, d1); + for (int c = 0; c < VDIM; c++) + { + Bu0[q][d2][c] += b * u0[d1][d2][c]; + Bu1[q][d2][c] += b * u1[d1][d2][c]; + } + } + } + } + real_t BBu0[max_Q1D][max_Q1D][VDIM]; + real_t BBu1[max_Q1D][max_Q1D][VDIM]; + for (int q1 = 0; q1 < Q1D; ++q1) + { + for (int q2 = 0; q2 < Q1D; q2++) + { + for (int c = 0; c < VDIM; c++) + { + BBu0[q1][q2][c] = 0.0; + BBu1[q1][q2][c] = 0.0; + } + for (int d2 = 0; d2 < D1D; ++d2) + { + const real_t b = B(q2, d2); + for (int c = 0; c < VDIM; c++) + { + BBu0[q1][q2][c] += b * Bu0[q1][d2][c]; + BBu1[q1][q2][c] += b * Bu1[q1][d2][c]; + } + } + } + } + real_t DBBu[max_Q1D][max_Q1D][VDIM]; + for (int q1 = 0; q1 < Q1D; ++q1) + { + for (int q2 = 0; q2 < Q1D; q2++) + { + for (int c = 0; c < VDIM; c++) + { + DBBu[q1][q2][c] = op(q1, q2, 0, 0, f) * BBu0[q1][q2][c] + + op(q1, q2, 1, 0, f) * BBu1[q1][q2][c]; + } + } + } + real_t BDBBu[max_Q1D][max_D1D][VDIM]; + for (int q1 = 0; q1 < Q1D; ++q1) + { + for (int d2 = 0; d2 < D1D; d2++) + { + for (int c = 0; c < VDIM; c++) + { + BDBBu[q1][d2][c] = 0.0; + } + for (int q2 = 0; q2 < Q1D; ++q2) + { + const real_t b = Bt(d2, q2); + for (int c = 0; c < VDIM; c++) + { + BDBBu[q1][d2][c] += b * DBBu[q1][q2][c]; + } + } + } + } + real_t BBDBBu[max_D1D][max_D1D][VDIM]; + for (int d1 = 0; d1 < D1D; ++d1) + { + for (int d2 = 0; d2 < D1D; d2++) + { + for (int c = 0; c < VDIM; c++) + { + BBDBBu[d1][d2][c] = 0.0; + } + for (int q1 = 0; q1 < Q1D; ++q1) + { + const real_t b = Bt(d1, q1); + for (int c = 0; c < VDIM; c++) + { + BBDBBu[d1][d2][c] += b * BDBBu[q1][d2][c]; + } + } + for (int c = 0; c < VDIM; c++) + { + y(d1, d2, c, 0, f) += BBDBBu[d1][d2][c]; + y(d1, d2, c, 1, f) += -BBDBBu[d1][d2][c]; + } + } + } + }); +} + +// Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein +template +static void SmemPADGTraceApply3D(const int NF, const Array &b, + const Array &bt, const Vector &op_, + const Vector &x_, Vector &y_, + const int d1d = 0, const int q1d = 0) +{ + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); + auto x = Reshape(x_.Read(), D1D, D1D, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF); + + mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f) + { + const int tidz = MFEM_THREAD_ID(z); + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D]; + MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D]; + MFEM_FOREACH_THREAD(d1, x, D1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + u0[tidz][d1][d2] = x(d1, d2, 0, f); + u1[tidz][d1][d2] = x(d1, d2, 1, f); + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D]; + MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + real_t Bu0_ = 0.0; + real_t Bu1_ = 0.0; + for (int d1 = 0; d1 < D1D; ++d1) + { + const real_t b = B(q1, d1); + Bu0_ += b * u0[tidz][d1][d2]; + Bu1_ += b * u1[tidz][d1][d2]; + } + Bu0[tidz][q1][d2] = Bu0_; + Bu1[tidz][q1][d2] = Bu1_; + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D]; + MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(q2, y, Q1D) + { + real_t BBu0_ = 0.0; + real_t BBu1_ = 0.0; + for (int d2 = 0; d2 < D1D; ++d2) + { + const real_t b = B(q2, d2); + BBu0_ += b * Bu0[tidz][q1][d2]; + BBu1_ += b * Bu1[tidz][q1][d2]; + } + BBu0[tidz][q1][q2] = BBu0_; + BBu1[tidz][q1][q2] = BBu1_; + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t DBBu[NBZ][max_Q1D][max_Q1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(q2, y, Q1D) + { + DBBu[tidz][q1][q2] = op(q1, q2, 0, 0, f) * BBu0[tidz][q1][q2] + + op(q1, q2, 1, 0, f) * BBu1[tidz][q1][q2]; + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t BDBBu[NBZ][max_Q1D][max_D1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + real_t BDBBu_ = 0.0; + for (int q2 = 0; q2 < Q1D; ++q2) + { + const real_t b = Bt(d2, q2); + BDBBu_ += b * DBBu[tidz][q1][q2]; + } + BDBBu[tidz][q1][d2] = BDBBu_; + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(d1, x, D1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + real_t BBDBBu_ = 0.0; + for (int q1 = 0; q1 < Q1D; ++q1) + { + const real_t b = Bt(d1, q1); + BBDBBu_ += b * BDBBu[tidz][q1][d2]; + } + y(d1, d2, 0, f) += BBDBBu_; + y(d1, d2, 1, f) += -BBDBBu_; + } + } + }); +} + +// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein +template +static void PADGTraceApplyTranspose2D(const int NF, const Array &b, + const Array &bt, + const Vector &op_, const Vector &x_, + Vector &y_, const int d1d = 0, + const int q1d = 0) +{ + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, 2, 2, NF); + auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF); + + mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) + { + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + real_t u0[max_D1D][VDIM]; + real_t u1[max_D1D][VDIM]; + for (int d = 0; d < D1D; d++) + { + for (int c = 0; c < VDIM; c++) + { + u0[d][c] = x(d, c, 0, f); + u1[d][c] = x(d, c, 1, f); + } + } + real_t Bu0[max_Q1D][VDIM]; + real_t Bu1[max_Q1D][VDIM]; + for (int q = 0; q < Q1D; ++q) + { + for (int c = 0; c < VDIM; c++) + { + Bu0[q][c] = 0.0; + Bu1[q][c] = 0.0; + } + for (int d = 0; d < D1D; ++d) + { + const real_t b = B(q, d); + for (int c = 0; c < VDIM; c++) + { + Bu0[q][c] += b * u0[d][c]; + Bu1[q][c] += b * u1[d][c]; + } + } + } + real_t DBu0[max_Q1D][VDIM]; + real_t DBu1[max_Q1D][VDIM]; + for (int q = 0; q < Q1D; ++q) + { + for (int c = 0; c < VDIM; c++) + { + DBu0[q][c] = + op(q, 0, 0, f) * Bu0[q][c] + op(q, 0, 1, f) * Bu1[q][c]; + DBu1[q][c] = + op(q, 1, 0, f) * Bu0[q][c] + op(q, 1, 1, f) * Bu1[q][c]; + } + } + real_t BDBu0[max_D1D][VDIM]; + real_t BDBu1[max_D1D][VDIM]; + for (int d = 0; d < D1D; ++d) + { + for (int c = 0; c < VDIM; c++) + { + BDBu0[d][c] = 0.0; + BDBu1[d][c] = 0.0; + } + for (int q = 0; q < Q1D; ++q) + { + const real_t b = Bt(d, q); + for (int c = 0; c < VDIM; c++) + { + BDBu0[d][c] += b * DBu0[q][c]; + BDBu1[d][c] += b * DBu1[q][c]; + } + } + for (int c = 0; c < VDIM; c++) + { + y(d, c, 0, f) += BDBu0[d][c]; + y(d, c, 1, f) += BDBu1[d][c]; + } + } + }); +} + +// PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein +template +static void PADGTraceApplyTranspose3D(const int NF, const Array &b, + const Array &bt, + const Vector &op_, const Vector &x_, + Vector &y_, const int d1d = 0, + const int q1d = 0) +{ + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); + auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF); + + mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) + { + const int VDIM = 1; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + real_t u0[max_D1D][max_D1D][VDIM]; + real_t u1[max_D1D][max_D1D][VDIM]; + for (int d1 = 0; d1 < D1D; d1++) + { + for (int d2 = 0; d2 < D1D; d2++) + { + for (int c = 0; c < VDIM; c++) + { + u0[d1][d2][c] = x(d1, d2, c, 0, f); + u1[d1][d2][c] = x(d1, d2, c, 1, f); + } + } + } + real_t Bu0[max_Q1D][max_D1D][VDIM]; + real_t Bu1[max_Q1D][max_D1D][VDIM]; + for (int q1 = 0; q1 < Q1D; ++q1) + { + for (int d2 = 0; d2 < D1D; ++d2) + { + for (int c = 0; c < VDIM; c++) + { + Bu0[q1][d2][c] = 0.0; + Bu1[q1][d2][c] = 0.0; + } + for (int d1 = 0; d1 < D1D; ++d1) + { + const real_t b = B(q1, d1); + for (int c = 0; c < VDIM; c++) + { + Bu0[q1][d2][c] += b * u0[d1][d2][c]; + Bu1[q1][d2][c] += b * u1[d1][d2][c]; + } + } + } + } + real_t BBu0[max_Q1D][max_Q1D][VDIM]; + real_t BBu1[max_Q1D][max_Q1D][VDIM]; + for (int q1 = 0; q1 < Q1D; ++q1) + { + for (int q2 = 0; q2 < Q1D; ++q2) + { + for (int c = 0; c < VDIM; c++) + { + BBu0[q1][q2][c] = 0.0; + BBu1[q1][q2][c] = 0.0; + } + for (int d2 = 0; d2 < D1D; ++d2) + { + const real_t b = B(q2, d2); + for (int c = 0; c < VDIM; c++) + { + BBu0[q1][q2][c] += b * Bu0[q1][d2][c]; + BBu1[q1][q2][c] += b * Bu1[q1][d2][c]; + } + } + } + } + real_t DBu0[max_Q1D][max_Q1D][VDIM]; + real_t DBu1[max_Q1D][max_Q1D][VDIM]; + for (int q1 = 0; q1 < Q1D; ++q1) + { + for (int q2 = 0; q2 < Q1D; ++q2) + { + const real_t D00 = op(q1, q2, 0, 0, f); + const real_t D01 = op(q1, q2, 0, 1, f); + const real_t D10 = op(q1, q2, 1, 0, f); + const real_t D11 = op(q1, q2, 1, 1, f); + for (int c = 0; c < VDIM; c++) + { + DBu0[q1][q2][c] = D00 * BBu0[q1][q2][c] + D01 * BBu1[q1][q2][c]; + DBu1[q1][q2][c] = D10 * BBu0[q1][q2][c] + D11 * BBu1[q1][q2][c]; + } + } + } + real_t BDBu0[max_D1D][max_Q1D][VDIM]; + real_t BDBu1[max_D1D][max_Q1D][VDIM]; + for (int d1 = 0; d1 < D1D; ++d1) + { + for (int q2 = 0; q2 < Q1D; ++q2) + { + for (int c = 0; c < VDIM; c++) + { + BDBu0[d1][q2][c] = 0.0; + BDBu1[d1][q2][c] = 0.0; + } + for (int q1 = 0; q1 < Q1D; ++q1) + { + const real_t b = Bt(d1, q1); + for (int c = 0; c < VDIM; c++) + { + BDBu0[d1][q2][c] += b * DBu0[q1][q2][c]; + BDBu1[d1][q2][c] += b * DBu1[q1][q2][c]; + } + } + } + } + real_t BBDBu0[max_D1D][max_D1D][VDIM]; + real_t BBDBu1[max_D1D][max_D1D][VDIM]; + for (int d1 = 0; d1 < D1D; ++d1) + { + for (int d2 = 0; d2 < D1D; ++d2) + { + for (int c = 0; c < VDIM; c++) + { + BBDBu0[d1][d2][c] = 0.0; + BBDBu1[d1][d2][c] = 0.0; + } + for (int q2 = 0; q2 < Q1D; ++q2) + { + const real_t b = Bt(d2, q2); + for (int c = 0; c < VDIM; c++) + { + BBDBu0[d1][d2][c] += b * BDBu0[d1][q2][c]; + BBDBu1[d1][d2][c] += b * BDBu1[d1][q2][c]; + } + } + for (int c = 0; c < VDIM; c++) + { + y(d1, d2, c, 0, f) += BBDBu0[d1][d2][c]; + y(d1, d2, c, 1, f) += BBDBu1[d1][d2][c]; + } + } + } + }); +} + +// Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein +template +static void SmemPADGTraceApplyTranspose3D(const int NF, const Array &b, + const Array &bt, + const Vector &op_, const Vector &x_, + Vector &y_, const int d1d = 0, + const int q1d = 0) +{ + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); + auto x = Reshape(x_.Read(), D1D, D1D, 2, NF); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF); + + mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f) + { + const int tidz = MFEM_THREAD_ID(z); + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + // the following variables are evaluated at compile time + constexpr int NBZ = T_NBZ ? T_NBZ : 1; + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D]; + MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D]; + MFEM_FOREACH_THREAD(d1, x, D1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + u0[tidz][d1][d2] = x(d1, d2, 0, f); + u1[tidz][d1][d2] = x(d1, d2, 1, f); + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D]; + MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + real_t Bu0_ = 0.0; + real_t Bu1_ = 0.0; + for (int d1 = 0; d1 < D1D; ++d1) + { + const real_t b = B(q1, d1); + Bu0_ += b * u0[tidz][d1][d2]; + Bu1_ += b * u1[tidz][d1][d2]; + } + Bu0[tidz][q1][d2] = Bu0_; + Bu1[tidz][q1][d2] = Bu1_; + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D]; + MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(q2, y, Q1D) + { + real_t BBu0_ = 0.0; + real_t BBu1_ = 0.0; + for (int d2 = 0; d2 < D1D; ++d2) + { + const real_t b = B(q2, d2); + BBu0_ += b * Bu0[tidz][q1][d2]; + BBu1_ += b * Bu1[tidz][q1][d2]; + } + BBu0[tidz][q1][q2] = BBu0_; + BBu1[tidz][q1][q2] = BBu1_; + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t DBBu0[NBZ][max_Q1D][max_Q1D]; + MFEM_SHARED real_t DBBu1[NBZ][max_Q1D][max_Q1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(q2, y, Q1D) + { + const real_t D00 = op(q1, q2, 0, 0, f); + const real_t D01 = op(q1, q2, 0, 1, f); + const real_t D10 = op(q1, q2, 1, 0, f); + const real_t D11 = op(q1, q2, 1, 1, f); + const real_t u0q = BBu0[tidz][q1][q2]; + const real_t u1q = BBu1[tidz][q1][q2]; + DBBu0[tidz][q1][q2] = D00 * u0q + D01 * u1q; + DBBu1[tidz][q1][q2] = D10 * u0q + D11 * u1q; + } + } + MFEM_SYNC_THREAD; + MFEM_SHARED real_t BDBBu0[NBZ][max_Q1D][max_D1D]; + MFEM_SHARED real_t BDBBu1[NBZ][max_Q1D][max_D1D]; + MFEM_FOREACH_THREAD(q1, x, Q1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + real_t BDBBu0_ = 0.0; + real_t BDBBu1_ = 0.0; + for (int q2 = 0; q2 < Q1D; ++q2) + { + const real_t b = Bt(d2, q2); + BDBBu0_ += b * DBBu0[tidz][q1][q2]; + BDBBu1_ += b * DBBu1[tidz][q1][q2]; + } + BDBBu0[tidz][q1][d2] = BDBBu0_; + BDBBu1[tidz][q1][d2] = BDBBu1_; + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(d1, x, D1D) + { + MFEM_FOREACH_THREAD(d2, y, D1D) + { + real_t BBDBBu0_ = 0.0; + real_t BBDBBu1_ = 0.0; + for (int q1 = 0; q1 < Q1D; ++q1) + { + const real_t b = Bt(d1, q1); + BBDBBu0_ += b * BDBBu0[tidz][q1][d2]; + BBDBBu1_ += b * BDBBu1[tidz][q1][d2]; + } + y(d1, d2, 0, f) += BBDBBu0_; + y(d1, d2, 1, f) += BBDBBu1_; + } + } + }); +} + +} // namespace internal + +template +DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPAKernels::Kernel() +{ + if constexpr (DIM == 2) + { + return internal::PADGTraceApply2D; + } + else if constexpr (DIM == 3) + { + if constexpr (D1D == 3 || D1D == 4) + { + return internal::SmemPADGTraceApply3D; + } + else + { + return internal::SmemPADGTraceApply3D; + } + } + MFEM_ABORT(""); +} + +template +DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPATKernels::Kernel() +{ + if constexpr (DIM == 2) + { + return internal::PADGTraceApplyTranspose2D; + } + else if constexpr (DIM == 3) + { + return internal::SmemPADGTraceApplyTranspose3D; + } + MFEM_ABORT(""); +} +} // namespace mfem + +/// \endcond DO_NOT_DOCUMENT +#endif diff --git a/fem/integ/bilininteg_dgtrace_pa.cpp b/fem/integ/bilininteg_dgtrace_pa.cpp index 4054129739..a74644e7c0 100644 --- a/fem/integ/bilininteg_dgtrace_pa.cpp +++ b/fem/integ/bilininteg_dgtrace_pa.cpp @@ -15,115 +15,102 @@ #include "../qfunction.hpp" #include "../restriction.hpp" +#include "bilininteg_dgtrace_kernels.hpp" + namespace mfem { // PA DG Trace Integrator -static void PADGTraceSetup2D(const int Q1D, - const int NF, - const Array &w, - const Vector &det, - const Vector &nor, - const Vector &rho, - const Vector &vel, - const real_t alpha, - const real_t beta, - Vector &op) +static void PADGTraceSetup2D(const int Q1D, const int NF, + const Array &w, const Vector &det, + const Vector &nor, const Vector &rho, + const Vector &vel, const real_t alpha, + const real_t beta, Vector &op) { const int VDIM = 2; auto d = Reshape(det.Read(), Q1D, NF); auto n = Reshape(nor.Read(), Q1D, VDIM, NF); const bool const_r = rho.Size() == 1; - auto R = - const_r ? Reshape(rho.Read(), 1,1) : Reshape(rho.Read(), Q1D,NF); + auto R = const_r ? Reshape(rho.Read(), 1, 1) : Reshape(rho.Read(), Q1D, NF); const bool const_v = vel.Size() == 2; auto V = - const_v ? Reshape(vel.Read(), 2,1,1) : Reshape(vel.Read(), 2,Q1D,NF); + const_v ? Reshape(vel.Read(), 2, 1, 1) : Reshape(vel.Read(), 2, Q1D, NF); auto W = w.Read(); auto qd = Reshape(op.Write(), Q1D, 2, 2, NF); - mfem::forall(Q1D*NF, [=] MFEM_HOST_DEVICE (int tid) + mfem::forall(Q1D * NF, [=] MFEM_HOST_DEVICE(int tid) { const int f = tid / Q1D; const int q = tid % Q1D; { - const real_t r = const_r ? R(0,0) : R(q,f); - const real_t v0 = const_v ? V(0,0,0) : V(0,q,f); - const real_t v1 = const_v ? V(1,0,0) : V(1,q,f); - const real_t dot = n(q,0,f) * v0 + n(q,1,f) * v1; + const real_t r = const_r ? R(0, 0) : R(q, f); + const real_t v0 = const_v ? V(0, 0, 0) : V(0, q, f); + const real_t v1 = const_v ? V(1, 0, 0) : V(1, q, f); + const real_t dot = n(q, 0, f) * v0 + n(q, 1, f) * v1; const real_t abs = dot > 0_r ? dot : -dot; - const real_t w = W[q]*r*d(q,f); - qd(q,0,0,f) = w*( alpha/2 * dot + beta * abs ); - qd(q,1,0,f) = w*( alpha/2 * dot - beta * abs ); - qd(q,0,1,f) = w*(-alpha/2 * dot - beta * abs ); - qd(q,1,1,f) = w*(-alpha/2 * dot + beta * abs ); + const real_t w = W[q] * r * d(q, f); + qd(q, 0, 0, f) = w * (alpha / 2 * dot + beta * abs); + qd(q, 1, 0, f) = w * (alpha / 2 * dot - beta * abs); + qd(q, 0, 1, f) = w * (-alpha / 2 * dot - beta * abs); + qd(q, 1, 1, f) = w * (-alpha / 2 * dot + beta * abs); } }); } -static void PADGTraceSetup3D(const int Q1D, - const int NF, - const Array &w, - const Vector &det, - const Vector &nor, - const Vector &rho, - const Vector &vel, - const real_t alpha, - const real_t beta, - Vector &op) +static void PADGTraceSetup3D(const int Q1D, const int NF, + const Array &w, const Vector &det, + const Vector &nor, const Vector &rho, + const Vector &vel, const real_t alpha, + const real_t beta, Vector &op) { const int VDIM = 3; auto d = Reshape(det.Read(), Q1D, Q1D, NF); auto n = Reshape(nor.Read(), Q1D, Q1D, VDIM, NF); const bool const_r = rho.Size() == 1; - auto R = - const_r ? Reshape(rho.Read(), 1,1,1) : Reshape(rho.Read(), Q1D,Q1D,NF); + auto R = const_r ? Reshape(rho.Read(), 1, 1, 1) + : Reshape(rho.Read(), Q1D, Q1D, NF); const bool const_v = vel.Size() == 3; - auto V = - const_v ? Reshape(vel.Read(), 3,1,1,1) : Reshape(vel.Read(), 3,Q1D,Q1D,NF); + auto V = const_v ? Reshape(vel.Read(), 3, 1, 1, 1) + : Reshape(vel.Read(), 3, Q1D, Q1D, NF); auto W = w.Read(); auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF); - mfem::forall(Q1D*Q1D*NF, [=] MFEM_HOST_DEVICE (int tid) + mfem::forall(Q1D * Q1D * NF, [=] MFEM_HOST_DEVICE(int tid) { int f = tid / (Q1D * Q1D); int q2 = (tid / Q1D) % Q1D; int q1 = tid % Q1D; { { - const real_t r = const_r ? R(0,0,0) : R(q1,q2,f); - const real_t v0 = const_v ? V(0,0,0,0) : V(0,q1,q2,f); - const real_t v1 = const_v ? V(1,0,0,0) : V(1,q1,q2,f); - const real_t v2 = const_v ? V(2,0,0,0) : V(2,q1,q2,f); - const real_t dot = n(q1,q2,0,f) * v0 + n(q1,q2,1,f) * v1 + - n(q1,q2,2,f) * v2; + const real_t r = const_r ? R(0, 0, 0) : R(q1, q2, f); + const real_t v0 = const_v ? V(0, 0, 0, 0) : V(0, q1, q2, f); + const real_t v1 = const_v ? V(1, 0, 0, 0) : V(1, q1, q2, f); + const real_t v2 = const_v ? V(2, 0, 0, 0) : V(2, q1, q2, f); + const real_t dot = n(q1, q2, 0, f) * v0 + n(q1, q2, 1, f) * v1 + + n(q1, q2, 2, f) * v2; const real_t abs = dot > 0.0 ? dot : -dot; - const real_t w = W[q1+q2*Q1D]*r*d(q1,q2,f); - qd(q1,q2,0,0,f) = w*( alpha/2 * dot + beta * abs ); - qd(q1,q2,1,0,f) = w*( alpha/2 * dot - beta * abs ); - qd(q1,q2,0,1,f) = w*(-alpha/2 * dot - beta * abs ); - qd(q1,q2,1,1,f) = w*(-alpha/2 * dot + beta * abs ); + const real_t w = W[q1 + q2 * Q1D] * r * d(q1, q2, f); + qd(q1, q2, 0, 0, f) = w * (alpha / 2 * dot + beta * abs); + qd(q1, q2, 1, 0, f) = w * (alpha / 2 * dot - beta * abs); + qd(q1, q2, 0, 1, f) = w * (-alpha / 2 * dot - beta * abs); + qd(q1, q2, 1, 1, f) = w * (-alpha / 2 * dot + beta * abs); } } }); } -static void PADGTraceSetup(const int dim, - const int D1D, - const int Q1D, - const int NF, - const Array &W, - const Vector &det, - const Vector &nor, - const Vector &rho, - const Vector &u, - const real_t alpha, - const real_t beta, - Vector &op) +static void PADGTraceSetup(const int dim, const int D1D, const int Q1D, + const int NF, const Array &W, + const Vector &det, const Vector &nor, + const Vector &rho, const Vector &u, + const real_t alpha, const real_t beta, Vector &op) { - if (dim == 1) { MFEM_ABORT("dim==1 not supported in PADGTraceSetup"); } + if (dim == 1) + { + MFEM_ABORT("dim==1 not supported in PADGTraceSetup"); + } if (dim == 2) { PADGTraceSetup2D(Q1D, NF, W, det, nor, rho, u, alpha, beta, op); @@ -136,11 +123,9 @@ static void PADGTraceSetup(const int dim, void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type) { - const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ? - Device::GetDeviceMemoryType() : pa_mt; + const MemoryType mt = + (pa_mt == MemoryType::DEFAULT) ? Device::GetDeviceMemoryType() : pa_mt; - nf = fes.GetNFbyType(type); - if (nf==0) { return; } // Assumes tensor-product elements Mesh *mesh = fes.GetMesh(); const FiniteElement &el = *fes.GetTypicalTraceElement(); @@ -148,19 +133,21 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type) IntRule: &GetRule(el.GetGeomType(), el.GetOrder(), *mesh->GetTypicalElementTransformation()); + + + FaceQuadratureSpace qs(*mesh, *ir, type); + nf = qs.GetNumFaces(); + if (nf==0) { return; } const int symmDims = 4; nq = ir->GetNPoints(); dim = mesh->Dimension(); geom = mesh->GetFaceGeometricFactors( - *ir, - FaceGeometricFactors::DETERMINANTS | - FaceGeometricFactors::NORMALS, type, mt); + *ir, FaceGeometricFactors::DETERMINANTS | FaceGeometricFactors::NORMALS, + type, mt); maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR); dofs1D = maps->ndof; quad1D = maps->nqpt; pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType()); - - FaceQuadratureSpace qs(*mesh, *ir, type); CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED); CoefficientVector r(qs, CoefficientStorage::COMPRESSED); @@ -168,13 +155,13 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type) { r.SetConstant(1.0); } - else if (ConstantCoefficient *const_rho = dynamic_cast - (rho)) + else if (ConstantCoefficient *const_rho = + dynamic_cast(rho)) { r.SetConstant(const_rho->constant); } - else if (QuadratureFunctionCoefficient* qf_rho = - dynamic_cast(rho)) + else if (QuadratureFunctionCoefficient *qf_rho = + dynamic_cast(rho)) { r.MakeRef(qf_rho->GetQuadFunction()); } @@ -199,8 +186,8 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type) for (int q = 0; q < nq; ++q) { // Convert to lexicographic ordering - int iq = ToLexOrdering(dim, face.element[0].local_face_id, - quad1D, q); + int iq = + ToLexOrdering(dim, face.element[0].local_face_id, quad1D, q); T.SetAllIntPoints(&ir->IntPoint(q)); const IntegrationPoint &eip1 = T.GetElement1IntPoint(); @@ -214,878 +201,134 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type) else { real_t udotn = 0.0; - for (int d=0; d= 0.0) + { + rq = rho->Eval(*T.Elem2, eip2); + } + else + { + rq = rho->Eval(*T.Elem1, eip1); } - if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); } - else { rq = rho->Eval(*T.Elem1, eip1); } } - C(iq,f_ind) = rq; + C(iq, f_ind) = rq; } f_ind++; } - MFEM_VERIFY(f_ind==nf, "Incorrect number of faces."); + MFEM_VERIFY(f_ind == nf, "Incorrect number of faces."); } - PADGTraceSetup(dim, dofs1D, quad1D, nf, ir->GetWeights(), - geom->detJ, geom->normal, r, vel, - alpha, beta, pa_data); + PADGTraceSetup(dim, dofs1D, quad1D, nf, ir->GetWeights(), geom->detJ, + geom->normal, r, vel, alpha, beta, pa_data); } -void DGTraceIntegrator::AssemblePAInteriorFaces(const FiniteElementSpace& fes) +void DGTraceIntegrator::AssemblePAInteriorFaces(const FiniteElementSpace &fes) { SetupPA(fes, FaceType::Interior); } -void DGTraceIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace& fes) +void DGTraceIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace &fes) { SetupPA(fes, FaceType::Boundary); } -// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein -template static -void PADGTraceApply2D(const int NF, - const Array &b, - const Array &bt, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, 2, 2, NF); - auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF); - - mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) - { - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - real_t u0[max_D1D][VDIM]; - real_t u1[max_D1D][VDIM]; - for (int d = 0; d < D1D; d++) - { - for (int c = 0; c < VDIM; c++) - { - u0[d][c] = x(d,c,0,f); - u1[d][c] = x(d,c,1,f); - } - } - real_t Bu0[max_Q1D][VDIM]; - real_t Bu1[max_Q1D][VDIM]; - for (int q = 0; q < Q1D; ++q) - { - for (int c = 0; c < VDIM; c++) - { - Bu0[q][c] = 0.0; - Bu1[q][c] = 0.0; - } - for (int d = 0; d < D1D; ++d) - { - const real_t b = B(q,d); - for (int c = 0; c < VDIM; c++) - { - Bu0[q][c] += b*u0[d][c]; - Bu1[q][c] += b*u1[d][c]; - } - } - } - real_t DBu[max_Q1D][VDIM]; - for (int q = 0; q < Q1D; ++q) - { - for (int c = 0; c < VDIM; c++) - { - DBu[q][c] = op(q,0,0,f)*Bu0[q][c] + op(q,1,0,f)*Bu1[q][c]; - } - } - real_t BDBu[max_D1D][VDIM]; - for (int d = 0; d < D1D; ++d) - { - for (int c = 0; c < VDIM; c++) - { - BDBu[d][c] = 0.0; - } - for (int q = 0; q < Q1D; ++q) - { - const real_t b = Bt(d,q); - for (int c = 0; c < VDIM; c++) - { - BDBu[d][c] += b*DBu[q][c]; - } - } - for (int c = 0; c < VDIM; c++) - { - y(d,c,0,f) += BDBu[d][c]; - y(d,c,1,f) += -BDBu[d][c]; - } - } - }); -} - -// PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein -template static -void PADGTraceApply3D(const int NF, - const Array &b, - const Array &bt, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); - auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF); - - mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) - { - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - real_t u0[max_D1D][max_D1D][VDIM]; - real_t u1[max_D1D][max_D1D][VDIM]; - for (int d1 = 0; d1 < D1D; d1++) - { - for (int d2 = 0; d2 < D1D; d2++) - { - for (int c = 0; c < VDIM; c++) - { - u0[d1][d2][c] = x(d1,d2,c,0,f); - u1[d1][d2][c] = x(d1,d2,c,1,f); - } - } - } - real_t Bu0[max_Q1D][max_D1D][VDIM]; - real_t Bu1[max_Q1D][max_D1D][VDIM]; - for (int q = 0; q < Q1D; ++q) - { - for (int d2 = 0; d2 < D1D; d2++) - { - for (int c = 0; c < VDIM; c++) - { - Bu0[q][d2][c] = 0.0; - Bu1[q][d2][c] = 0.0; - } - for (int d1 = 0; d1 < D1D; ++d1) - { - const real_t b = B(q,d1); - for (int c = 0; c < VDIM; c++) - { - Bu0[q][d2][c] += b*u0[d1][d2][c]; - Bu1[q][d2][c] += b*u1[d1][d2][c]; - } - } - } - } - real_t BBu0[max_Q1D][max_Q1D][VDIM]; - real_t BBu1[max_Q1D][max_Q1D][VDIM]; - for (int q1 = 0; q1 < Q1D; ++q1) - { - for (int q2 = 0; q2 < Q1D; q2++) - { - for (int c = 0; c < VDIM; c++) - { - BBu0[q1][q2][c] = 0.0; - BBu1[q1][q2][c] = 0.0; - } - for (int d2 = 0; d2 < D1D; ++d2) - { - const real_t b = B(q2,d2); - for (int c = 0; c < VDIM; c++) - { - BBu0[q1][q2][c] += b*Bu0[q1][d2][c]; - BBu1[q1][q2][c] += b*Bu1[q1][d2][c]; - } - } - } - } - real_t DBBu[max_Q1D][max_Q1D][VDIM]; - for (int q1 = 0; q1 < Q1D; ++q1) - { - for (int q2 = 0; q2 < Q1D; q2++) - { - for (int c = 0; c < VDIM; c++) - { - DBBu[q1][q2][c] = op(q1,q2,0,0,f)*BBu0[q1][q2][c] + - op(q1,q2,1,0,f)*BBu1[q1][q2][c]; - } - } - } - real_t BDBBu[max_Q1D][max_D1D][VDIM]; - for (int q1 = 0; q1 < Q1D; ++q1) - { - for (int d2 = 0; d2 < D1D; d2++) - { - for (int c = 0; c < VDIM; c++) - { - BDBBu[q1][d2][c] = 0.0; - } - for (int q2 = 0; q2 < Q1D; ++q2) - { - const real_t b = Bt(d2,q2); - for (int c = 0; c < VDIM; c++) - { - BDBBu[q1][d2][c] += b*DBBu[q1][q2][c]; - } - } - } - } - real_t BBDBBu[max_D1D][max_D1D][VDIM]; - for (int d1 = 0; d1 < D1D; ++d1) - { - for (int d2 = 0; d2 < D1D; d2++) - { - for (int c = 0; c < VDIM; c++) - { - BBDBBu[d1][d2][c] = 0.0; - } - for (int q1 = 0; q1 < Q1D; ++q1) - { - const real_t b = Bt(d1,q1); - for (int c = 0; c < VDIM; c++) - { - BBDBBu[d1][d2][c] += b*BDBBu[q1][d2][c]; - } - } - for (int c = 0; c < VDIM; c++) - { - y(d1,d2,c,0,f) += BBDBBu[d1][d2][c]; - y(d1,d2,c,1,f) += -BBDBBu[d1][d2][c]; - } - } - } - }); -} - -// Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein -template static -void SmemPADGTraceApply3D(const int NF, - const Array &b, - const Array &bt, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); - auto x = Reshape(x_.Read(), D1D, D1D, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF); - - mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int f) - { - const int tidz = MFEM_THREAD_ID(z); - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D]; - MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D]; - MFEM_FOREACH_THREAD(d1,x,D1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - u0[tidz][d1][d2] = x(d1,d2,0,f); - u1[tidz][d1][d2] = x(d1,d2,1,f); - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D]; - MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - real_t Bu0_ = 0.0; - real_t Bu1_ = 0.0; - for (int d1 = 0; d1 < D1D; ++d1) - { - const real_t b = B(q1,d1); - Bu0_ += b*u0[tidz][d1][d2]; - Bu1_ += b*u1[tidz][d1][d2]; - } - Bu0[tidz][q1][d2] = Bu0_; - Bu1[tidz][q1][d2] = Bu1_; - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D]; - MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(q2,y,Q1D) - { - real_t BBu0_ = 0.0; - real_t BBu1_ = 0.0; - for (int d2 = 0; d2 < D1D; ++d2) - { - const real_t b = B(q2,d2); - BBu0_ += b*Bu0[tidz][q1][d2]; - BBu1_ += b*Bu1[tidz][q1][d2]; - } - BBu0[tidz][q1][q2] = BBu0_; - BBu1[tidz][q1][q2] = BBu1_; - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t DBBu[NBZ][max_Q1D][max_Q1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(q2,y,Q1D) - { - DBBu[tidz][q1][q2] = op(q1,q2,0,0,f)*BBu0[tidz][q1][q2] + - op(q1,q2,1,0,f)*BBu1[tidz][q1][q2]; - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t BDBBu[NBZ][max_Q1D][max_D1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - real_t BDBBu_ = 0.0; - for (int q2 = 0; q2 < Q1D; ++q2) - { - const real_t b = Bt(d2,q2); - BDBBu_ += b*DBBu[tidz][q1][q2]; - } - BDBBu[tidz][q1][d2] = BDBBu_; - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(d1,x,D1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - real_t BBDBBu_ = 0.0; - for (int q1 = 0; q1 < Q1D; ++q1) - { - const real_t b = Bt(d1,q1); - BBDBBu_ += b*BDBBu[tidz][q1][d2]; - } - y(d1,d2,0,f) += BBDBBu_; - y(d1,d2,1,f) += -BBDBBu_; - } - } - }); -} - -static void PADGTraceApply(const int dim, - const int D1D, - const int Q1D, - const int NF, - const Array &B, - const Array &Bt, - const Vector &op, - const Vector &x, - Vector &y) -{ - if (dim == 2) - { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return PADGTraceApply2D<2,2>(NF,B,Bt,op,x,y); - case 0x33: return PADGTraceApply2D<3,3>(NF,B,Bt,op,x,y); - case 0x44: return PADGTraceApply2D<4,4>(NF,B,Bt,op,x,y); - case 0x55: return PADGTraceApply2D<5,5>(NF,B,Bt,op,x,y); - case 0x66: return PADGTraceApply2D<6,6>(NF,B,Bt,op,x,y); - case 0x77: return PADGTraceApply2D<7,7>(NF,B,Bt,op,x,y); - case 0x88: return PADGTraceApply2D<8,8>(NF,B,Bt,op,x,y); - case 0x99: return PADGTraceApply2D<9,9>(NF,B,Bt,op,x,y); - default: return PADGTraceApply2D(NF,B,Bt,op,x,y,D1D,Q1D); - } - } - else if (dim == 3) - { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return SmemPADGTraceApply3D<2,2,1>(NF,B,Bt,op,x,y); - case 0x23: return SmemPADGTraceApply3D<2,3,1>(NF,B,Bt,op,x,y); - case 0x34: return SmemPADGTraceApply3D<3,4,2>(NF,B,Bt,op,x,y); - case 0x45: return SmemPADGTraceApply3D<4,5,2>(NF,B,Bt,op,x,y); - case 0x56: return SmemPADGTraceApply3D<5,6,1>(NF,B,Bt,op,x,y); - case 0x67: return SmemPADGTraceApply3D<6,7,1>(NF,B,Bt,op,x,y); - case 0x78: return SmemPADGTraceApply3D<7,8,1>(NF,B,Bt,op,x,y); - case 0x89: return SmemPADGTraceApply3D<8,9,1>(NF,B,Bt,op,x,y); - default: return PADGTraceApply3D(NF,B,Bt,op,x,y,D1D,Q1D); - } - } - MFEM_ABORT("Unknown kernel."); -} - -// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein -template static -void PADGTraceApplyTranspose2D(const int NF, - const Array &b, - const Array &bt, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, 2, 2, NF); - auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF); - - mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) - { - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - real_t u0[max_D1D][VDIM]; - real_t u1[max_D1D][VDIM]; - for (int d = 0; d < D1D; d++) - { - for (int c = 0; c < VDIM; c++) - { - u0[d][c] = x(d,c,0,f); - u1[d][c] = x(d,c,1,f); - } - } - real_t Bu0[max_Q1D][VDIM]; - real_t Bu1[max_Q1D][VDIM]; - for (int q = 0; q < Q1D; ++q) - { - for (int c = 0; c < VDIM; c++) - { - Bu0[q][c] = 0.0; - Bu1[q][c] = 0.0; - } - for (int d = 0; d < D1D; ++d) - { - const real_t b = B(q,d); - for (int c = 0; c < VDIM; c++) - { - Bu0[q][c] += b*u0[d][c]; - Bu1[q][c] += b*u1[d][c]; - } - } - } - real_t DBu0[max_Q1D][VDIM]; - real_t DBu1[max_Q1D][VDIM]; - for (int q = 0; q < Q1D; ++q) - { - for (int c = 0; c < VDIM; c++) - { - DBu0[q][c] = op(q,0,0,f)*Bu0[q][c] + op(q,0,1,f)*Bu1[q][c]; - DBu1[q][c] = op(q,1,0,f)*Bu0[q][c] + op(q,1,1,f)*Bu1[q][c]; - } - } - real_t BDBu0[max_D1D][VDIM]; - real_t BDBu1[max_D1D][VDIM]; - for (int d = 0; d < D1D; ++d) - { - for (int c = 0; c < VDIM; c++) - { - BDBu0[d][c] = 0.0; - BDBu1[d][c] = 0.0; - } - for (int q = 0; q < Q1D; ++q) - { - const real_t b = Bt(d,q); - for (int c = 0; c < VDIM; c++) - { - BDBu0[d][c] += b*DBu0[q][c]; - BDBu1[d][c] += b*DBu1[q][c]; - } - } - for (int c = 0; c < VDIM; c++) - { - y(d,c,0,f) += BDBu0[d][c]; - y(d,c,1,f) += BDBu1[d][c]; - } - } - }); -} - -// PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein -template static -void PADGTraceApplyTranspose3D(const int NF, - const Array &b, - const Array &bt, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); - auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF); - - mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) - { - const int VDIM = 1; - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - real_t u0[max_D1D][max_D1D][VDIM]; - real_t u1[max_D1D][max_D1D][VDIM]; - for (int d1 = 0; d1 < D1D; d1++) - { - for (int d2 = 0; d2 < D1D; d2++) - { - for (int c = 0; c < VDIM; c++) - { - u0[d1][d2][c] = x(d1,d2,c,0,f); - u1[d1][d2][c] = x(d1,d2,c,1,f); - } - } - } - real_t Bu0[max_Q1D][max_D1D][VDIM]; - real_t Bu1[max_Q1D][max_D1D][VDIM]; - for (int q1 = 0; q1 < Q1D; ++q1) - { - for (int d2 = 0; d2 < D1D; ++d2) - { - for (int c = 0; c < VDIM; c++) - { - Bu0[q1][d2][c] = 0.0; - Bu1[q1][d2][c] = 0.0; - } - for (int d1 = 0; d1 < D1D; ++d1) - { - const real_t b = B(q1,d1); - for (int c = 0; c < VDIM; c++) - { - Bu0[q1][d2][c] += b*u0[d1][d2][c]; - Bu1[q1][d2][c] += b*u1[d1][d2][c]; - } - } - } - } - real_t BBu0[max_Q1D][max_Q1D][VDIM]; - real_t BBu1[max_Q1D][max_Q1D][VDIM]; - for (int q1 = 0; q1 < Q1D; ++q1) - { - for (int q2 = 0; q2 < Q1D; ++q2) - { - for (int c = 0; c < VDIM; c++) - { - BBu0[q1][q2][c] = 0.0; - BBu1[q1][q2][c] = 0.0; - } - for (int d2 = 0; d2 < D1D; ++d2) - { - const real_t b = B(q2,d2); - for (int c = 0; c < VDIM; c++) - { - BBu0[q1][q2][c] += b*Bu0[q1][d2][c]; - BBu1[q1][q2][c] += b*Bu1[q1][d2][c]; - } - } - } - } - real_t DBu0[max_Q1D][max_Q1D][VDIM]; - real_t DBu1[max_Q1D][max_Q1D][VDIM]; - for (int q1 = 0; q1 < Q1D; ++q1) - { - for (int q2 = 0; q2 < Q1D; ++q2) - { - const real_t D00 = op(q1,q2,0,0,f); - const real_t D01 = op(q1,q2,0,1,f); - const real_t D10 = op(q1,q2,1,0,f); - const real_t D11 = op(q1,q2,1,1,f); - for (int c = 0; c < VDIM; c++) - { - DBu0[q1][q2][c] = D00*BBu0[q1][q2][c] + D01*BBu1[q1][q2][c]; - DBu1[q1][q2][c] = D10*BBu0[q1][q2][c] + D11*BBu1[q1][q2][c]; - } - } - } - real_t BDBu0[max_D1D][max_Q1D][VDIM]; - real_t BDBu1[max_D1D][max_Q1D][VDIM]; - for (int d1 = 0; d1 < D1D; ++d1) - { - for (int q2 = 0; q2 < Q1D; ++q2) - { - for (int c = 0; c < VDIM; c++) - { - BDBu0[d1][q2][c] = 0.0; - BDBu1[d1][q2][c] = 0.0; - } - for (int q1 = 0; q1 < Q1D; ++q1) - { - const real_t b = Bt(d1,q1); - for (int c = 0; c < VDIM; c++) - { - BDBu0[d1][q2][c] += b*DBu0[q1][q2][c]; - BDBu1[d1][q2][c] += b*DBu1[q1][q2][c]; - } - } - } - } - real_t BBDBu0[max_D1D][max_D1D][VDIM]; - real_t BBDBu1[max_D1D][max_D1D][VDIM]; - for (int d1 = 0; d1 < D1D; ++d1) - { - for (int d2 = 0; d2 < D1D; ++d2) - { - for (int c = 0; c < VDIM; c++) - { - BBDBu0[d1][d2][c] = 0.0; - BBDBu1[d1][d2][c] = 0.0; - } - for (int q2 = 0; q2 < Q1D; ++q2) - { - const real_t b = Bt(d2,q2); - for (int c = 0; c < VDIM; c++) - { - BBDBu0[d1][d2][c] += b*BDBu0[d1][q2][c]; - BBDBu1[d1][d2][c] += b*BDBu1[d1][q2][c]; - } - } - for (int c = 0; c < VDIM; c++) - { - y(d1,d2,c,0,f) += BBDBu0[d1][d2][c]; - y(d1,d2,c,1,f) += BBDBu1[d1][d2][c]; - } - } - } - }); -} - -// Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein -template static -void SmemPADGTraceApplyTranspose3D(const int NF, - const Array &b, - const Array &bt, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF); - auto x = Reshape(x_.Read(), D1D, D1D, 2, NF); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF); - - mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int f) - { - const int tidz = MFEM_THREAD_ID(z); - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int NBZ = T_NBZ ? T_NBZ : 1; - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D]; - MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D]; - MFEM_FOREACH_THREAD(d1,x,D1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - u0[tidz][d1][d2] = x(d1,d2,0,f); - u1[tidz][d1][d2] = x(d1,d2,1,f); - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D]; - MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - real_t Bu0_ = 0.0; - real_t Bu1_ = 0.0; - for (int d1 = 0; d1 < D1D; ++d1) - { - const real_t b = B(q1,d1); - Bu0_ += b*u0[tidz][d1][d2]; - Bu1_ += b*u1[tidz][d1][d2]; - } - Bu0[tidz][q1][d2] = Bu0_; - Bu1[tidz][q1][d2] = Bu1_; - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D]; - MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(q2,y,Q1D) - { - real_t BBu0_ = 0.0; - real_t BBu1_ = 0.0; - for (int d2 = 0; d2 < D1D; ++d2) - { - const real_t b = B(q2,d2); - BBu0_ += b*Bu0[tidz][q1][d2]; - BBu1_ += b*Bu1[tidz][q1][d2]; - } - BBu0[tidz][q1][q2] = BBu0_; - BBu1[tidz][q1][q2] = BBu1_; - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t DBBu0[NBZ][max_Q1D][max_Q1D]; - MFEM_SHARED real_t DBBu1[NBZ][max_Q1D][max_Q1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(q2,y,Q1D) - { - const real_t D00 = op(q1,q2,0,0,f); - const real_t D01 = op(q1,q2,0,1,f); - const real_t D10 = op(q1,q2,1,0,f); - const real_t D11 = op(q1,q2,1,1,f); - const real_t u0q = BBu0[tidz][q1][q2]; - const real_t u1q = BBu1[tidz][q1][q2]; - DBBu0[tidz][q1][q2] = D00*u0q + D01*u1q; - DBBu1[tidz][q1][q2] = D10*u0q + D11*u1q; - } - } - MFEM_SYNC_THREAD; - MFEM_SHARED real_t BDBBu0[NBZ][max_Q1D][max_D1D]; - MFEM_SHARED real_t BDBBu1[NBZ][max_Q1D][max_D1D]; - MFEM_FOREACH_THREAD(q1,x,Q1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - real_t BDBBu0_ = 0.0; - real_t BDBBu1_ = 0.0; - for (int q2 = 0; q2 < Q1D; ++q2) - { - const real_t b = Bt(d2,q2); - BDBBu0_ += b*DBBu0[tidz][q1][q2]; - BDBBu1_ += b*DBBu1[tidz][q1][q2]; - } - BDBBu0[tidz][q1][d2] = BDBBu0_; - BDBBu1[tidz][q1][d2] = BDBBu1_; - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(d1,x,D1D) - { - MFEM_FOREACH_THREAD(d2,y,D1D) - { - real_t BBDBBu0_ = 0.0; - real_t BBDBBu1_ = 0.0; - for (int q1 = 0; q1 < Q1D; ++q1) - { - const real_t b = Bt(d1,q1); - BBDBBu0_ += b*BDBBu0[tidz][q1][d2]; - BBDBBu1_ += b*BDBBu1[tidz][q1][d2]; - } - y(d1,d2,0,f) += BBDBBu0_; - y(d1,d2,1,f) += BBDBBu1_; - } - } - }); -} - -static void PADGTraceApplyTranspose(const int dim, - const int D1D, - const int Q1D, - const int NF, - const Array &B, - const Array &Bt, - const Vector &op, - const Vector &x, - Vector &y) -{ - if (dim == 2) - { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return PADGTraceApplyTranspose2D<2,2>(NF,B,Bt,op,x,y); - case 0x33: return PADGTraceApplyTranspose2D<3,3>(NF,B,Bt,op,x,y); - case 0x44: return PADGTraceApplyTranspose2D<4,4>(NF,B,Bt,op,x,y); - case 0x55: return PADGTraceApplyTranspose2D<5,5>(NF,B,Bt,op,x,y); - case 0x66: return PADGTraceApplyTranspose2D<6,6>(NF,B,Bt,op,x,y); - case 0x77: return PADGTraceApplyTranspose2D<7,7>(NF,B,Bt,op,x,y); - case 0x88: return PADGTraceApplyTranspose2D<8,8>(NF,B,Bt,op,x,y); - case 0x99: return PADGTraceApplyTranspose2D<9,9>(NF,B,Bt,op,x,y); - default: return PADGTraceApplyTranspose2D(NF,B,Bt,op,x,y,D1D,Q1D); - } - } - else if (dim == 3) - { - switch ((D1D << 4 ) | Q1D) - { - case 0x22: return SmemPADGTraceApplyTranspose3D<2,2>(NF,B,Bt,op,x,y); - case 0x23: return SmemPADGTraceApplyTranspose3D<2,3>(NF,B,Bt,op,x,y); - case 0x34: return SmemPADGTraceApplyTranspose3D<3,4>(NF,B,Bt,op,x,y); - case 0x45: return SmemPADGTraceApplyTranspose3D<4,5>(NF,B,Bt,op,x,y); - case 0x56: return SmemPADGTraceApplyTranspose3D<5,6>(NF,B,Bt,op,x,y); - case 0x67: return SmemPADGTraceApplyTranspose3D<6,7>(NF,B,Bt,op,x,y); - case 0x78: return SmemPADGTraceApplyTranspose3D<7,8>(NF,B,Bt,op,x,y); - case 0x89: return SmemPADGTraceApplyTranspose3D<8,9>(NF,B,Bt,op,x,y); - default: return PADGTraceApplyTranspose3D(NF,B,Bt,op,x,y,D1D,Q1D); - } - } - MFEM_ABORT("Unknown kernel."); -} - // PA DGTraceIntegrator Apply kernel void DGTraceIntegrator::AddMultPA(const Vector &x, Vector &y) const { - PADGTraceApply(dim, dofs1D, quad1D, nf, - maps->B, maps->Bt, - pa_data, x, y); + ApplyPAKernels::Run(dim, dofs1D, quad1D, nf, maps->B, maps->Bt, pa_data, x, + y, dofs1D, quad1D); } void DGTraceIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const { - PADGTraceApplyTranspose(dim, dofs1D, quad1D, nf, - maps->B, maps->Bt, - pa_data, x, y); + ApplyPATKernels::Run(dim, dofs1D, quad1D, nf, maps->B, maps->Bt, pa_data, x, + y, dofs1D, quad1D); } +DGTraceIntegrator::DGTraceIntegrator(real_t a, real_t b) : alpha(a), beta(b) +{ + static Kernels kernels; +} + +DGTraceIntegrator::DGTraceIntegrator(VectorCoefficient &u_, real_t a) + : DGTraceIntegrator(a, 0.5 * a) +{ + u = &u_; +} + +DGTraceIntegrator::DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b) + : DGTraceIntegrator(a, b) +{ + u = &u_; +} + +DGTraceIntegrator::DGTraceIntegrator(Coefficient &rho_, VectorCoefficient &u_, + real_t a, real_t b) + : DGTraceIntegrator(a, b) +{ + rho = &rho_; + u = &u_; +} + +/// \cond DO_NOT_DOCUMENT + +DGTraceIntegrator::Kernels::Kernels() +{ + // 2D + DGTraceIntegrator::AddSpecialization<2, 2, 2>(); + DGTraceIntegrator::AddSpecialization<2, 3, 3>(); + DGTraceIntegrator::AddSpecialization<2, 4, 4>(); + DGTraceIntegrator::AddSpecialization<2, 5, 5>(); + DGTraceIntegrator::AddSpecialization<2, 6, 6>(); + DGTraceIntegrator::AddSpecialization<2, 7, 7>(); + DGTraceIntegrator::AddSpecialization<2, 8, 8>(); + DGTraceIntegrator::AddSpecialization<2, 9, 9>(); + // 3D + DGTraceIntegrator::AddSpecialization<3, 2, 3>(); + DGTraceIntegrator::AddSpecialization<3, 3, 4>(); + DGTraceIntegrator::AddSpecialization<3, 4, 5>(); + DGTraceIntegrator::AddSpecialization<3, 5, 6>(); + DGTraceIntegrator::AddSpecialization<3, 6, 7>(); + DGTraceIntegrator::AddSpecialization<3, 7, 8>(); + DGTraceIntegrator::AddSpecialization<3, 8, 9>(); +} + +DGTraceIntegrator::ApplyKernelType +DGTraceIntegrator::ApplyPAKernels::Fallback(int dim, int, int) +{ + if (dim == 2) + { + return internal::PADGTraceApply2D; + } + else if (dim == 3) + { + return internal::PADGTraceApply3D; + } + else + { + MFEM_ABORT(""); + } +} + +DGTraceIntegrator::ApplyKernelType +DGTraceIntegrator::ApplyPATKernels::Fallback(int dim, int, int) +{ + if (dim == 2) + { + return internal::PADGTraceApplyTranspose2D; + } + else if (dim == 3) + { + return internal::PADGTraceApplyTranspose3D; + } + else + { + MFEM_ABORT(""); + } +} + +/// \endcond DO_NOT_DOCUMENT + } // namespace mfem diff --git a/fem/integ/bilininteg_diffusion_kernels.cpp b/fem/integ/bilininteg_diffusion_kernels.cpp index 8ddaf148b1..901e6d7d95 100644 --- a/fem/integ/bilininteg_diffusion_kernels.cpp +++ b/fem/integ/bilininteg_diffusion_kernels.cpp @@ -19,6 +19,8 @@ namespace mfem DiffusionIntegrator::Kernels::Kernels() { // 2D + // Q = P+1 + DiffusionIntegrator::AddSpecialization<2,1,1>(); DiffusionIntegrator::AddSpecialization<2,2,2>(); DiffusionIntegrator::AddSpecialization<2,3,3>(); DiffusionIntegrator::AddSpecialization<2,4,4>(); @@ -27,17 +29,39 @@ DiffusionIntegrator::Kernels::Kernels() DiffusionIntegrator::AddSpecialization<2,7,7>(); DiffusionIntegrator::AddSpecialization<2,8,8>(); DiffusionIntegrator::AddSpecialization<2,9,9>(); + // Q = P+2 + DiffusionIntegrator::AddSpecialization<2,1,2>(); + DiffusionIntegrator::AddSpecialization<2,2,3>(); + DiffusionIntegrator::AddSpecialization<2,3,4>(); + DiffusionIntegrator::AddSpecialization<2,4,5>(); + DiffusionIntegrator::AddSpecialization<2,5,6>(); + DiffusionIntegrator::AddSpecialization<2,6,7>(); + DiffusionIntegrator::AddSpecialization<2,7,8>(); + DiffusionIntegrator::AddSpecialization<2,8,9>(); + DiffusionIntegrator::AddSpecialization<2,9,10>(); + // others // 3D + // Q = P+1 + DiffusionIntegrator::AddSpecialization<3,1,1>(); DiffusionIntegrator::AddSpecialization<3,2,2>(); + DiffusionIntegrator::AddSpecialization<3,3,3>(); + DiffusionIntegrator::AddSpecialization<3,4,4>(); + DiffusionIntegrator::AddSpecialization<3,5,5>(); + DiffusionIntegrator::AddSpecialization<3,6,6>(); + DiffusionIntegrator::AddSpecialization<3,7,7>(); + DiffusionIntegrator::AddSpecialization<3,8,8>(); + // Q = P+2 + DiffusionIntegrator::AddSpecialization<3,1,2>(); DiffusionIntegrator::AddSpecialization<3,2,3>(); DiffusionIntegrator::AddSpecialization<3,3,4>(); DiffusionIntegrator::AddSpecialization<3,4,5>(); - DiffusionIntegrator::AddSpecialization<3,4,6>(); DiffusionIntegrator::AddSpecialization<3,5,6>(); - DiffusionIntegrator::AddSpecialization<3,5,8>(); DiffusionIntegrator::AddSpecialization<3,6,7>(); DiffusionIntegrator::AddSpecialization<3,7,8>(); DiffusionIntegrator::AddSpecialization<3,8,9>(); + // others + DiffusionIntegrator::AddSpecialization<3,4,6>(); + DiffusionIntegrator::AddSpecialization<3,5,8>(); } namespace internal diff --git a/fem/integ/bilininteg_diffusion_kernels.hpp b/fem/integ/bilininteg_diffusion_kernels.hpp index 61dc0374e4..d7541fa3cf 100644 --- a/fem/integ/bilininteg_diffusion_kernels.hpp +++ b/fem/integ/bilininteg_diffusion_kernels.hpp @@ -672,12 +672,12 @@ inline void SmemPADiffusionApply2D(const int NE, real_t (*Gt)[MQ1] = (real_t (*)[MQ1]) (sBG+1); MFEM_SHARED real_t Xz[NBZ][MD1][MD1]; MFEM_SHARED real_t GD[2][NBZ][MD1][MQ1]; - MFEM_SHARED real_t GQ[2][NBZ][MD1][MQ1]; + MFEM_SHARED real_t GQ[2][NBZ][MQ1][MQ1]; real_t (*X)[MD1] = (real_t (*)[MD1])(Xz + tidz); - real_t (*DQ0)[MD1] = (real_t (*)[MD1])(GD[0] + tidz); - real_t (*DQ1)[MD1] = (real_t (*)[MD1])(GD[1] + tidz); - real_t (*QQ0)[MD1] = (real_t (*)[MD1])(GQ[0] + tidz); - real_t (*QQ1)[MD1] = (real_t (*)[MD1])(GQ[1] + tidz); + real_t (*DQ0)[MQ1] = (real_t (*)[MQ1])(GD[0] + tidz); + real_t (*DQ1)[MQ1] = (real_t (*)[MQ1])(GD[1] + tidz); + real_t (*QQ0)[MQ1] = (real_t (*)[MQ1])(GQ[0] + tidz); + real_t (*QQ1)[MQ1] = (real_t (*)[MQ1])(GQ[1] + tidz); MFEM_FOREACH_THREAD(dy,y,D1D) { MFEM_FOREACH_THREAD(dx,x,D1D) @@ -1221,9 +1221,9 @@ using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType; template ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel() { - if (DIM == 2) { return internal::SmemPADiffusionApply2D; } - else if (DIM == 3) { return internal::SmemPADiffusionApply3D; } - else { MFEM_ABORT(""); } + if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D; } + else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D; } + MFEM_ABORT(""); } inline @@ -1237,9 +1237,9 @@ ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int) template DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel() { - if (DIM == 2) { return internal::SmemPADiffusionDiagonal2D; } - else if (DIM == 3) { return internal::SmemPADiffusionDiagonal3D; } - else { MFEM_ABORT(""); } + if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D; } + else if constexpr (DIM == 3) { return internal::SmemPADiffusionDiagonal3D; } + MFEM_ABORT(""); } inline DiagonalKernelType diff --git a/fem/integ/bilininteg_hcurl_kernels.cpp b/fem/integ/bilininteg_hcurl_kernels.cpp index 50d82276ca..9846bc1cc9 100644 --- a/fem/integ/bilininteg_hcurl_kernels.cpp +++ b/fem/integ/bilininteg_hcurl_kernels.cpp @@ -599,13 +599,11 @@ void PACurlCurlSetup3D(const int Q1D, }); } -void PACurlCurlAssembleDiagonal2D(const int D1D, - const int Q1D, - const int NE, - const Array &bo, +void PACurlCurlAssembleDiagonal2D(const int D1D, const int Q1D, const bool, + const int NE, const Array &bo, + const Array &, const Array &, const Array &gc, - const Vector &pa_data, - Vector &diag) + const Vector &pa_data, Vector &diag) { auto Bo = Reshape(bo.Read(), Q1D, D1D-1); auto Gc = Reshape(gc.Read(), Q1D, D1D); @@ -653,16 +651,11 @@ void PACurlCurlAssembleDiagonal2D(const int D1D, }); // end of element loop } -void PACurlCurlApply2D(const int D1D, - const int Q1D, - const int NE, - const Array &bo, - const Array &bot, - const Array &gc, - const Array &gct, - const Vector &pa_data, - const Vector &x, - Vector &y, +void PACurlCurlApply2D(const int D1D, const int Q1D, const bool, const int NE, + const Array &bo, const Array &, + const Array &bot, const Array &, + const Array &gc, const Array &gct, + const Vector &pa_data, const Vector &x, Vector &y, const bool useAbs) { diff --git a/fem/integ/bilininteg_hcurl_kernels.hpp b/fem/integ/bilininteg_hcurl_kernels.hpp index 5ce6a7cb0c..815e6844e1 100644 --- a/fem/integ/bilininteg_hcurl_kernels.hpp +++ b/fem/integ/bilininteg_hcurl_kernels.hpp @@ -24,7 +24,7 @@ namespace mfem { - +/// \cond DO_NOT_DOCUMENT namespace internal { @@ -426,8 +426,11 @@ void PACurlCurlSetup3D(const int Q1D, // PA H(curl) curl-curl Diagonal 2D kernel void PACurlCurlAssembleDiagonal2D(const int D1D, const int Q1D, + const bool symmetric, // unused const int NE, const Array &bo, + const Array &bc, // unused + const Array &go, // unused const Array &gc, const Vector &pa_data, Vector &diag); @@ -831,9 +834,12 @@ inline void SmemPACurlCurlAssembleDiagonal3D(const int d1d, // PA H(curl) curl-curl Apply/AbsApply 2D kernel void PACurlCurlApply2D(const int D1D, const int Q1D, + const bool symmetric, // unused const int NE, const Array &bo, + const Array &bc, // unused const Array &bot, + const Array &bct, // unused const Array &gc, const Array &gct, const Vector &pa_data, @@ -3158,6 +3164,49 @@ inline void SmemPAHcurlL2ApplyTranspose3D(const int d1d, } // namespace internal +template +CurlCurlIntegrator::ApplyKernelType CurlCurlIntegrator::ApplyPAKernels::Kernel() +{ + if constexpr (DIM == 2) + { + return internal::PACurlCurlApply2D; + } + else if constexpr (DIM == 3) + { + if (Device::Allows(Backend::DEVICE_MASK)) + { + return internal::SmemPACurlCurlApply3D; + } + else + { + return internal::PACurlCurlApply3D; + } + } + MFEM_ABORT(""); +} + +template +CurlCurlIntegrator::DiagonalKernelType +CurlCurlIntegrator::DiagonalPAKernels::Kernel() +{ + if constexpr (DIM == 2) + { + return internal::PACurlCurlAssembleDiagonal2D; + } + else if constexpr (DIM == 3) + { + if (Device::Allows(Backend::DEVICE_MASK)) + { + return internal::SmemPACurlCurlAssembleDiagonal3D; + } + else + { + return internal::PACurlCurlAssembleDiagonal3D; + } + } + MFEM_ABORT(""); +} +/// \endcond DO_NOT_DOCUMENT } // namespace mfem #endif diff --git a/fem/integ/bilininteg_hcurlhdiv_kernels.hpp b/fem/integ/bilininteg_hcurlhdiv_kernels.hpp index ba10c7a27c..4d10e5274e 100644 --- a/fem/integ/bilininteg_hcurlhdiv_kernels.hpp +++ b/fem/integ/bilininteg_hcurlhdiv_kernels.hpp @@ -19,6 +19,7 @@ #include "../../linalg/vector.hpp" #include "../bilininteg.hpp" +/// \cond DO_NOT_DOCUMENT namespace mfem { @@ -819,4 +820,6 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d, } // namespace mfem +/// \endcond DO_NOT_DOCUMENT + #endif diff --git a/fem/integ/bilininteg_mass_kernels.cpp b/fem/integ/bilininteg_mass_kernels.cpp index 55251b580b..62c543ab26 100644 --- a/fem/integ/bilininteg_mass_kernels.cpp +++ b/fem/integ/bilininteg_mass_kernels.cpp @@ -17,6 +17,8 @@ namespace mfem MassIntegrator::Kernels::Kernels() { // 2D + // Q=P+1 + MassIntegrator::AddSpecialization<2,1,1>(); MassIntegrator::AddSpecialization<2,2,2>(); MassIntegrator::AddSpecialization<2,3,3>(); MassIntegrator::AddSpecialization<2,4,4>(); @@ -25,17 +27,45 @@ MassIntegrator::Kernels::Kernels() MassIntegrator::AddSpecialization<2,7,7>(); MassIntegrator::AddSpecialization<2,8,8>(); MassIntegrator::AddSpecialization<2,9,9>(); + // Q=P+2 + MassIntegrator::AddSpecialization<2,1,2>(); + MassIntegrator::AddSpecialization<2,2,3>(); + MassIntegrator::AddSpecialization<2,3,4>(); + MassIntegrator::AddSpecialization<2,4,5>(); + MassIntegrator::AddSpecialization<2,5,6>(); + MassIntegrator::AddSpecialization<2,6,7>(); + MassIntegrator::AddSpecialization<2,7,8>(); + MassIntegrator::AddSpecialization<2,8,9>(); + MassIntegrator::AddSpecialization<2,9,10>(); + // others + MassIntegrator::AddSpecialization<2,2,4>(); + MassIntegrator::AddSpecialization<2,3,6>(); + MassIntegrator::AddSpecialization<2,4,6>(); // 3D + // Q=P+1 + MassIntegrator::AddSpecialization<3,1,1>(); MassIntegrator::AddSpecialization<3,2,2>(); + MassIntegrator::AddSpecialization<3,3,3>(); + MassIntegrator::AddSpecialization<3,4,4>(); + MassIntegrator::AddSpecialization<3,5,5>(); + MassIntegrator::AddSpecialization<3,6,6>(); + MassIntegrator::AddSpecialization<3,7,7>(); + MassIntegrator::AddSpecialization<3,8,8>(); + MassIntegrator::AddSpecialization<3,9,9>(); + // Q=P+2 + MassIntegrator::AddSpecialization<3,1,2>(); MassIntegrator::AddSpecialization<3,2,3>(); MassIntegrator::AddSpecialization<3,3,4>(); + MassIntegrator::AddSpecialization<3,3,6>(); MassIntegrator::AddSpecialization<3,4,5>(); - MassIntegrator::AddSpecialization<3,4,6>(); MassIntegrator::AddSpecialization<3,5,6>(); - MassIntegrator::AddSpecialization<3,5,8>(); MassIntegrator::AddSpecialization<3,6,7>(); MassIntegrator::AddSpecialization<3,7,8>(); MassIntegrator::AddSpecialization<3,8,9>(); + // others + MassIntegrator::AddSpecialization<3,2,4>(); + MassIntegrator::AddSpecialization<3,4,6>(); + MassIntegrator::AddSpecialization<3,5,8>(); } namespace internal diff --git a/fem/integ/bilininteg_mass_kernels.hpp b/fem/integ/bilininteg_mass_kernels.hpp index e3a44b8d10..ecc875e13b 100644 --- a/fem/integ/bilininteg_mass_kernels.hpp +++ b/fem/integ/bilininteg_mass_kernels.hpp @@ -1392,10 +1392,10 @@ using DiagonalKernelType = MassIntegrator::DiagonalKernelType; template ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel() { - if (DIM == 1) { return internal::PAMassApply1D; } - else if (DIM == 2) { return internal::SmemPAMassApply2D; } - else if (DIM == 3) { return internal::SmemPAMassApply3D; } - else { MFEM_ABORT(""); } + if constexpr (DIM == 1) { return internal::PAMassApply1D; } + else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D; } + else if constexpr (DIM == 3) { return internal::SmemPAMassApply3D; } + MFEM_ABORT(""); } inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback( @@ -1410,10 +1410,10 @@ inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback( template DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel() { - if (DIM == 1) { return internal::PAMassAssembleDiagonal1D; } - else if (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D; } - else if (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D; } - else { MFEM_ABORT(""); } + if constexpr (DIM == 1) { return internal::PAMassAssembleDiagonal1D; } + else if constexpr (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D; } + else if constexpr (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D; } + MFEM_ABORT(""); } inline DiagonalKernelType MassIntegrator::DiagonalPAKernels::Fallback( diff --git a/fem/integ/bilininteg_mass_pa.cpp b/fem/integ/bilininteg_mass_pa.cpp index 1385da6d70..c18677a360 100644 --- a/fem/integ/bilininteg_mass_pa.cpp +++ b/fem/integ/bilininteg_mass_pa.cpp @@ -59,26 +59,23 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes) QuadratureSpace qs(*mesh, *ir); CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED); - - const int NE = ne; - const int Q1D = quad1D; - const int NQ = static_cast(std::pow(Q1D, dim)); - const bool const_c = coeff.Size() == 1; - const bool by_val = map_type == FiniteElement::VALUE; - const auto W = Reshape(ir->GetWeights().Read(), NQ); - const auto J = Reshape(geom->detJ.Read(), NQ, NE); - const auto C = const_c ? Reshape(coeff.Read(), 1, 1) : - Reshape(coeff.Read(), NQ,NE); - auto v = Reshape(pa_data.Write(), NQ, NE); - mfem::forall_2D(NE, NQ, 1, [=] MFEM_HOST_DEVICE (int e) { - MFEM_FOREACH_THREAD(i, x, NQ) + const int NE = ne; + const int NQ = nq; + const bool const_c = coeff.Size() == 1; + const bool by_val = map_type == FiniteElement::VALUE; + const auto W = Reshape(ir->GetWeights().Read(), NQ); + const auto J = Reshape(geom->detJ.Read(), NQ, NE); + const auto C = + const_c ? Reshape(coeff.Read(), 1, 1) : Reshape(coeff.Read(), NQ, NE); + auto v = Reshape(pa_data.Write(), NQ, NE); + mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e) { - const real_t detJ = J(i,e); - const real_t coeff = const_c ? C(0,0) : C(i,e); - v(i,e) = W(i) * coeff * (by_val ? detJ : 1.0/detJ); - } - }); + const real_t detJ = J(q, e); + const real_t coeff = const_c ? C(0, 0) : C(q, e); + v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ); + }); + } } void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes) @@ -109,50 +106,22 @@ void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes) CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED); const int NE = ne; - const int Q1D = quad1D; + const int NQ = nq; const bool const_c = coeff.Size() == 1; const bool by_val = map_type == FiniteElement::VALUE; - if (dim==1) { - const auto W = Reshape(ir->GetWeights().Read(), Q1D); - const auto J = Reshape(face_geom->detJ.Read(), Q1D, NE); - const auto C = const_c ? Reshape(coeff.Read(), 1, 1) : - Reshape(coeff.Read(), Q1D, NE); - auto v = Reshape(pa_data.Write(), Q1D, NE); - mfem::forall_2D(NE, Q1D, 1, [=] MFEM_HOST_DEVICE (int e) + const auto W = Reshape(ir->GetWeights().Read(), NQ); + const auto J = Reshape(face_geom->detJ.Read(), NQ, NE); + const auto C = const_c ? Reshape(coeff.Read(), 1, 1) + : Reshape(coeff.Read(), NQ, NE); + auto v = Reshape(pa_data.Write(), NQ, NE); + mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e) { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - const real_t detJ = J(qx,e); - const real_t coeff = const_c ? C(0,0) : C(qx,e); - v(qx,e) = W(qx) * coeff * (by_val ? detJ : 1.0/detJ); - } + const real_t detJ = J(q, e); + const real_t coeff = const_c ? C(0, 0) : C(q, e); + v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ); }); } - else if (dim==2) - { - const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D); - const auto J = Reshape(face_geom->detJ.Read(), Q1D,Q1D,NE); - const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) : - Reshape(coeff.Read(), Q1D,Q1D,NE); - auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE); - mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e) - { - MFEM_FOREACH_THREAD(qx,x,Q1D) - { - MFEM_FOREACH_THREAD(qy,y,Q1D) - { - const real_t detJ = J(qx,qy,e); - const real_t coeff = const_c ? C(0,0,0) : C(qx,qy,e); - v(qx,qy,e) = W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ); - } - } - }); - } - else - { - MFEM_ABORT("Not supported."); - } } void MassIntegrator::AssembleDiagonalPA(Vector &diag) diff --git a/fem/integ/bilininteg_vecdiffusion_kernels.hpp b/fem/integ/bilininteg_vecdiffusion_kernels.hpp new file mode 100644 index 0000000000..b93bb59527 --- /dev/null +++ b/fem/integ/bilininteg_vecdiffusion_kernels.hpp @@ -0,0 +1,338 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_BILININTEG_VECDIFFUSION_KERNELS_HPP +#define MFEM_BILININTEG_VECDIFFUSION_KERNELS_HPP + +#include "../../general/forall.hpp" +#include "../bilininteg.hpp" +#include "../ceed/integrators/diffusion/diffusion.hpp" +#include "../gridfunc.hpp" +#include "../qfunction.hpp" + +/// \cond DO_NOT_DOCUMENT +namespace mfem::internal +{ + +// PA Diffusion Apply 2D kernel +template +static void +PAVectorDiffusionApply2D(const int NE, const Array &b, + const Array &g, const Array &bt, + const Array >, const Vector &d_, + const Vector &x_, Vector &y_, const int d1d = 0, + const int q1d = 0, const int vdim = 0) +{ + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + const int VDIM = T_VDIM ? T_VDIM : vdim; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto G = Reshape(g.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto Gt = Reshape(gt.Read(), D1D, Q1D); + auto D = Reshape(d_.Read(), Q1D * Q1D, 3, NE); + auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + const int VDIM = T_VDIM ? T_VDIM : vdim; + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + + real_t grad[max_Q1D][max_Q1D][2]; + for (int c = 0; c < VDIM; c++) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + grad[qy][qx][0] = 0.0; + grad[qy][qx][1] = 0.0; + } + } + for (int dy = 0; dy < D1D; ++dy) + { + real_t gradX[max_Q1D][2]; + for (int qx = 0; qx < Q1D; ++qx) + { + gradX[qx][0] = 0.0; + gradX[qx][1] = 0.0; + } + for (int dx = 0; dx < D1D; ++dx) + { + const real_t s = x(dx, dy, c, e); + for (int qx = 0; qx < Q1D; ++qx) + { + gradX[qx][0] += s * B(qx, dx); + gradX[qx][1] += s * G(qx, dx); + } + } + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t wy = B(qy, dy); + const real_t wDy = G(qy, dy); + for (int qx = 0; qx < Q1D; ++qx) + { + grad[qy][qx][0] += gradX[qx][1] * wy; + grad[qy][qx][1] += gradX[qx][0] * wDy; + } + } + } + // Calculate Dxy, xDy in plane + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + const int q = qx + qy * Q1D; + const real_t O11 = D(q, 0, e); + const real_t O12 = D(q, 1, e); + const real_t O22 = D(q, 2, e); + const real_t gradX = grad[qy][qx][0]; + const real_t gradY = grad[qy][qx][1]; + grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY); + grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY); + } + } + for (int qy = 0; qy < Q1D; ++qy) + { + real_t gradX[max_D1D][2]; + for (int dx = 0; dx < D1D; ++dx) + { + gradX[dx][0] = 0.0; + gradX[dx][1] = 0.0; + } + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t gX = grad[qy][qx][0]; + const real_t gY = grad[qy][qx][1]; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t wx = Bt(dx, qx); + const real_t wDx = Gt(dx, qx); + gradX[dx][0] += gX * wDx; + gradX[dx][1] += gY * wx; + } + } + for (int dy = 0; dy < D1D; ++dy) + { + const real_t wy = Bt(dy, qy); + const real_t wDy = Gt(dy, qy); + for (int dx = 0; dx < D1D; ++dx) + { + y(dx, dy, c, e) += + ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy)); + } + } + } + } + }); +} + +// PA Diffusion Apply 3D kernel +template +static void +PAVectorDiffusionApply3D(const int NE, const Array &b, + const Array &g, const Array &bt, + const Array >, const Vector &op_, + const Vector &x_, Vector &y_, const int d1d = 0, + const int q1d = 0, const int sdim = 0) +{ + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + constexpr int VDIM = 3; + MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); + MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); + auto B = Reshape(b.Read(), Q1D, D1D); + auto G = Reshape(g.Read(), Q1D, D1D); + auto Bt = Reshape(bt.Read(), D1D, Q1D); + auto Gt = Reshape(gt.Read(), D1D, Q1D); + auto op = Reshape(op_.Read(), Q1D * Q1D * Q1D, 6, NE); + auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE); + auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e) + { + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + for (int c = 0; c < VDIM; ++c) + { + real_t grad[max_Q1D][max_Q1D][max_Q1D][3]; + for (int qz = 0; qz < Q1D; ++qz) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + grad[qz][qy][qx][0] = 0.0; + grad[qz][qy][qx][1] = 0.0; + grad[qz][qy][qx][2] = 0.0; + } + } + } + for (int dz = 0; dz < D1D; ++dz) + { + real_t gradXY[max_Q1D][max_Q1D][3]; + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + gradXY[qy][qx][0] = 0.0; + gradXY[qy][qx][1] = 0.0; + gradXY[qy][qx][2] = 0.0; + } + } + for (int dy = 0; dy < D1D; ++dy) + { + real_t gradX[max_Q1D][2]; + for (int qx = 0; qx < Q1D; ++qx) + { + gradX[qx][0] = 0.0; + gradX[qx][1] = 0.0; + } + for (int dx = 0; dx < D1D; ++dx) + { + const real_t s = x(dx, dy, dz, c, e); + for (int qx = 0; qx < Q1D; ++qx) + { + gradX[qx][0] += s * B(qx, dx); + gradX[qx][1] += s * G(qx, dx); + } + } + for (int qy = 0; qy < Q1D; ++qy) + { + const real_t wy = B(qy, dy); + const real_t wDy = G(qy, dy); + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t wx = gradX[qx][0]; + const real_t wDx = gradX[qx][1]; + gradXY[qy][qx][0] += wDx * wy; + gradXY[qy][qx][1] += wx * wDy; + gradXY[qy][qx][2] += wx * wy; + } + } + } + for (int qz = 0; qz < Q1D; ++qz) + { + const real_t wz = B(qz, dz); + const real_t wDz = G(qz, dz); + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz; + grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz; + grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz; + } + } + } + } + // Calculate Dxyz, xDyz, xyDz in plane + for (int qz = 0; qz < Q1D; ++qz) + { + for (int qy = 0; qy < Q1D; ++qy) + { + for (int qx = 0; qx < Q1D; ++qx) + { + const int q = qx + (qy + qz * Q1D) * Q1D; + const real_t O11 = op(q, 0, e); + const real_t O12 = op(q, 1, e); + const real_t O13 = op(q, 2, e); + const real_t O22 = op(q, 3, e); + const real_t O23 = op(q, 4, e); + const real_t O33 = op(q, 5, e); + const real_t gradX = grad[qz][qy][qx][0]; + const real_t gradY = grad[qz][qy][qx][1]; + const real_t gradZ = grad[qz][qy][qx][2]; + grad[qz][qy][qx][0] = + (O11 * gradX) + (O12 * gradY) + (O13 * gradZ); + grad[qz][qy][qx][1] = + (O12 * gradX) + (O22 * gradY) + (O23 * gradZ); + grad[qz][qy][qx][2] = + (O13 * gradX) + (O23 * gradY) + (O33 * gradZ); + } + } + } + for (int qz = 0; qz < Q1D; ++qz) + { + real_t gradXY[max_D1D][max_D1D][3]; + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + gradXY[dy][dx][0] = 0; + gradXY[dy][dx][1] = 0; + gradXY[dy][dx][2] = 0; + } + } + for (int qy = 0; qy < Q1D; ++qy) + { + real_t gradX[max_D1D][3]; + for (int dx = 0; dx < D1D; ++dx) + { + gradX[dx][0] = 0; + gradX[dx][1] = 0; + gradX[dx][2] = 0; + } + for (int qx = 0; qx < Q1D; ++qx) + { + const real_t gX = grad[qz][qy][qx][0]; + const real_t gY = grad[qz][qy][qx][1]; + const real_t gZ = grad[qz][qy][qx][2]; + for (int dx = 0; dx < D1D; ++dx) + { + const real_t wx = Bt(dx, qx); + const real_t wDx = Gt(dx, qx); + gradX[dx][0] += gX * wDx; + gradX[dx][1] += gY * wx; + gradX[dx][2] += gZ * wx; + } + } + for (int dy = 0; dy < D1D; ++dy) + { + const real_t wy = Bt(dy, qy); + const real_t wDy = Gt(dy, qy); + for (int dx = 0; dx < D1D; ++dx) + { + gradXY[dy][dx][0] += gradX[dx][0] * wy; + gradXY[dy][dx][1] += gradX[dx][1] * wDy; + gradXY[dy][dx][2] += gradX[dx][2] * wy; + } + } + } + for (int dz = 0; dz < D1D; ++dz) + { + const real_t wz = Bt(dz, qz); + const real_t wDz = Gt(dz, qz); + for (int dy = 0; dy < D1D; ++dy) + { + for (int dx = 0; dx < D1D; ++dx) + { + y(dx, dy, dz, c, e) += + ((gradXY[dy][dx][0] * wz) + (gradXY[dy][dx][1] * wz) + + (gradXY[dy][dx][2] * wDz)); + } + } + } + } + } + }); +} +} // namespace mfem::internal + +/// \endcond DO_NOT_DOCUMENT + +#endif // MFEM_BILININTEG_VECDIFFUSION_KERNELS_HPP diff --git a/fem/integ/bilininteg_vecdiffusion_pa.cpp b/fem/integ/bilininteg_vecdiffusion_pa.cpp index 0356c803c6..6a811e126f 100644 --- a/fem/integ/bilininteg_vecdiffusion_pa.cpp +++ b/fem/integ/bilininteg_vecdiffusion_pa.cpp @@ -9,219 +9,322 @@ // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. -#include "../../general/forall.hpp" #include "../bilininteg.hpp" -#include "../gridfunc.hpp" -#include "../qfunction.hpp" +#include "../../general/forall.hpp" #include "../ceed/integrators/diffusion/diffusion.hpp" +#include "./bilininteg_vecdiffusion_pa.hpp" // IWYU pragma: keep + +// #include "bilininteg_vecdiffusion_kernels.hpp" +// #include "bilininteg_vecdiffusion_pa.hpp" + namespace mfem { -// PA Diffusion Assemble 2D kernel -static void PAVectorDiffusionSetup2D(const int Q1D, - const int NE, - const Array &w, - const Vector &j, - const Vector &c, - Vector &op) +VectorDiffusionIntegrator::VectorDiffusionIntegrator(const IntegrationRule *ir) + : BilinearFormIntegrator(ir) { - const int NQ = Q1D*Q1D; - auto W = w.Read(); - - auto J = Reshape(j.Read(), NQ, 2, 2, NE); - auto y = Reshape(op.Write(), NQ, 3, NE); - - const bool const_c = c.Size() == 1; - const auto C = const_c ? Reshape(c.Read(), 1,1) : - Reshape(c.Read(), NQ, NE); - - - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - for (int q = 0; q < NQ; ++q) - { - const real_t J11 = J(q,0,0,e); - const real_t J21 = J(q,1,0,e); - const real_t J12 = J(q,0,1,e); - const real_t J22 = J(q,1,1,e); - - const real_t C1 = const_c ? C(0,0) : C(q,e); - const real_t c_detJ = W[q] * C1 / ((J11*J22)-(J21*J12)); - y(q,0,e) = c_detJ * (J12*J12 + J22*J22); // 1,1 - y(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2 - y(q,2,e) = c_detJ * (J11*J11 + J21*J21); // 2,2 - } - }); + // static Kernels kernels; } -// PA Diffusion Assemble 3D kernel -static void PAVectorDiffusionSetup3D(const int Q1D, - const int NE, - const Array &w, - const Vector &j, - const Vector &c, - Vector &op) +VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q) + : VectorDiffusionIntegrator() { - const int NQ = Q1D*Q1D*Q1D; - auto W = w.Read(); - auto J = Reshape(j.Read(), NQ, 3, 3, NE); - auto y = Reshape(op.Write(), NQ, 6, NE); - - const bool const_c = c.Size() == 1; - const auto C = const_c ? Reshape(c.Read(), 1,1) : - Reshape(c.Read(), NQ,NE); - - - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - for (int q = 0; q < NQ; ++q) - { - const real_t J11 = J(q,0,0,e); - const real_t J21 = J(q,1,0,e); - const real_t J31 = J(q,2,0,e); - const real_t J12 = J(q,0,1,e); - const real_t J22 = J(q,1,1,e); - const real_t J32 = J(q,2,1,e); - const real_t J13 = J(q,0,2,e); - const real_t J23 = J(q,1,2,e); - const real_t J33 = J(q,2,2,e); - const real_t detJ = J11 * (J22 * J33 - J32 * J23) - - J21 * (J12 * J33 - J32 * J13) + - J31 * (J12 * J23 - J22 * J13); - - const real_t C1 = const_c ? C(0,0) : C(q,e); - - const real_t c_detJ = W[q] * C1 / detJ; - // adj(J) - const real_t A11 = (J22 * J33) - (J23 * J32); - const real_t A12 = (J32 * J13) - (J12 * J33); - const real_t A13 = (J12 * J23) - (J22 * J13); - const real_t A21 = (J31 * J23) - (J21 * J33); - const real_t A22 = (J11 * J33) - (J13 * J31); - const real_t A23 = (J21 * J13) - (J11 * J23); - const real_t A31 = (J21 * J32) - (J31 * J22); - const real_t A32 = (J31 * J12) - (J11 * J32); - const real_t A33 = (J11 * J22) - (J12 * J21); - // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T - y(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1 - y(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1 - y(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1 - y(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2 - y(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2 - y(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3 - } - }); + Q = &q; } -static void PAVectorDiffusionSetup(const int dim, - const int Q1D, - const int NE, - const Array &W, - const Vector &J, - const Vector &C, - Vector &op) +VectorDiffusionIntegrator::VectorDiffusionIntegrator(int vector_dimension) + : VectorDiffusionIntegrator() { - if (!(dim == 2 || dim == 3)) - { - MFEM_ABORT("Dimension not supported."); - } - if (dim == 2) - { - PAVectorDiffusionSetup2D(Q1D, NE, W, J, C, op); - } - if (dim == 3) - { - PAVectorDiffusionSetup3D(Q1D, NE, W, J, C, op); - } + vdim = vector_dimension; +} + +VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q, + const IntegrationRule *ir) + : VectorDiffusionIntegrator(ir) +{ + Q = &q; +} + +VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q, + int vector_dimension) + : VectorDiffusionIntegrator() +{ + Q = &q; + vdim = vector_dimension; +} + +VectorDiffusionIntegrator::VectorDiffusionIntegrator(VectorCoefficient &vq) + : VectorDiffusionIntegrator() +{ + VQ = &vq; + vdim = vq.GetVDim(); +} + +VectorDiffusionIntegrator::VectorDiffusionIntegrator(MatrixCoefficient &mq) + : VectorDiffusionIntegrator() +{ + MQ = &mq; + vdim = mq.GetVDim(); } void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes) { - // Assumes tensor-product elements Mesh *mesh = fes.GetMesh(); const FiniteElement &el = *fes.GetTypicalFE(); - const IntegrationRule *ir - = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el); + const auto *ir = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el); + if (DeviceCanUseCeed()) { delete ceedOp; - const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 || - fes.IsVariableOrder(); - if (mixed) - { - ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q); - } - else - { - ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q); - } + const bool mixed = + mesh->GetNumGeometries(mesh->Dimension()) > 1 || fes.IsVariableOrder(); + if (mixed) { ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q); } + else { ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q); } return; } - const int dims = el.GetDim(); - const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6 - const int nq = ir->GetNPoints(); + + // If vdim is not set, set it to the space dimension + vdim = (vdim == -1) ? fes.GetVDim() : vdim; + MFEM_VERIFY(vdim == fes.GetVDim(), "vdim != fes.GetVDim()"); + + const MemoryType mt = pa_mt == MemoryType::DEFAULT + ? Device::GetDeviceMemoryType() + : pa_mt; + + ne = fes.GetNE(); dim = mesh->Dimension(); sdim = mesh->SpaceDimension(); - ne = fes.GetNE(); - geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS); + const int nq = ir->GetNPoints(); + geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt); maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR); dofs1D = maps->ndof; quad1D = maps->nqpt; - pa_data.SetSize(symmDims * nq * ne, Device::GetDeviceMemoryType()); + const int q1d = quad1D; - MFEM_VERIFY(!VQ && !MQ, - "Only scalar coefficient supported for partial assembly for VectorDiffusionIntegrator"); + if (!(dim == 2 || dim == 3)) { MFEM_ABORT("Dimension not supported."); } QuadratureSpace qs(*mesh, *ir); - CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED); + CoefficientVector coeff(qs, CoefficientStorage::FULL); + + if (Q) + { + coeff.Project(*Q); + } + else if (VQ) + { + coeff.Project(*VQ); + MFEM_VERIFY(VQ->GetVDim() == vdim, "VQ vdim vs. vdim error"); + } + else if (MQ) + { + coeff.ProjectTranspose(*MQ); + MFEM_VERIFY(MQ->GetVDim() == vdim, "MQ dimension vs. vdim error"); + MFEM_VERIFY(coeff.Size() == (vdim*vdim) * ne * nq, "MQ size error"); + } + else { coeff.SetConstant(1.0); } + + coeff_vdim = coeff.GetVDim(); + const bool scalar_coeff = coeff_vdim == 1; + const bool vector_coeff = coeff_vdim == vdim; + const bool matrix_coeff = coeff_vdim == vdim * vdim; + MFEM_VERIFY(scalar_coeff + vector_coeff + matrix_coeff == 1, ""); + + const int pa_size = dim * dim; + pa_data.SetSize(nq * pa_size * vdim * (matrix_coeff ? dim : 1) * ne, mt); - const Array &w = ir->GetWeights(); - const Vector &j = geom->J; - Vector &d = pa_data; - if (dim == 1) { MFEM_ABORT("dim==1 not supported in PAVectorDiffusionSetup"); } if (dim == 2 && sdim == 3) { - constexpr int DIM = 2; - constexpr int SDIM = 3; - const int NQ = quad1D*quad1D; - auto W = w.Read(); - auto J = Reshape(j.Read(), NQ, SDIM, DIM, ne); - auto D = Reshape(d.Write(), NQ, SDIM, ne); + MFEM_VERIFY(scalar_coeff, ""); + const int nc = vdim; + const auto W = Reshape(ir->GetWeights().Read(), q1d, q1d); + const auto J = Reshape(geom->J.Read(), q1d, q1d, sdim, dim, ne); + const auto C = Reshape(coeff.Read(), coeff_vdim, q1d, q1d, ne); + auto D = Reshape(pa_data.Write(), q1d, q1d, pa_size, + vdim * (matrix_coeff ? dim : 1), ne); - const bool const_c = coeff.Size() == 1; - const auto C = const_c ? Reshape(coeff.Read(), 1,1) : - Reshape(coeff.Read(), NQ,ne); - - mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e) + mfem::forall_2D(ne, q1d, q1d, [=] MFEM_HOST_DEVICE(int e) { - for (int q = 0; q < NQ; ++q) + MFEM_FOREACH_THREAD(qy, y, q1d) { - const real_t wq = W[q]; - const real_t J11 = J(q,0,0,e); - const real_t J21 = J(q,1,0,e); - const real_t J31 = J(q,2,0,e); - const real_t J12 = J(q,0,1,e); - const real_t J22 = J(q,1,1,e); - const real_t J32 = J(q,2,1,e); - const real_t E = J11*J11 + J21*J21 + J31*J31; - const real_t G = J12*J12 + J22*J22 + J32*J32; - const real_t F = J11*J12 + J21*J22 + J31*J32; - const real_t iw = 1.0 / sqrt(E*G - F*F); - const real_t C1 = const_c ? C(0,0) : C(q,e); - const real_t alpha = wq * C1 * iw; - D(q,0,e) = alpha * G; // 1,1 - D(q,1,e) = -alpha * F; // 1,2 - D(q,2,e) = alpha * E; // 2,2 + MFEM_FOREACH_THREAD(qx, x, q1d) + { + for (int i = 0; i < nc; ++i) + { + const real_t wq = W(qx, qy); + const real_t J11 = J(qx, qy, 0, 0, e); + const real_t J21 = J(qx, qy, 1, 0, e); + const real_t J31 = J(qx, qy, 2, 0, e); + const real_t J12 = J(qx, qy, 0, 1, e); + const real_t J22 = J(qx, qy, 1, 1, e); + const real_t J32 = J(qx, qy, 2, 1, e); + const real_t E = J11*J11 + J21*J21 + J31*J31; + const real_t G = J12*J12 + J22*J22 + J32*J32; + const real_t F = J11*J12 + J21*J22 + J31*J32; + const real_t iw = 1.0 / sqrt(E*G - F*F); + const auto C0 = C(0, qx, qy, e); + const real_t alpha = wq * C0 * iw; + D(qx, qy, 0, i, e) = alpha * G; // 1,1 + D(qx, qy, 1, i, e) = -alpha * F; // 1,2 + D(qx, qy, 2, i, e) = -alpha * F; // 2,1 == 1,2 + D(qx, qy, 3, i, e) = alpha * E; // 2,2 + } + } + } + }); + } + else if (dim == 2 && sdim == 2) + { + const int nc = vdim, cvdim = coeff_vdim; + const auto W = Reshape(ir->GetWeights().Read(), q1d, q1d); + const auto J = Reshape(geom->J.Read(), q1d, q1d, sdim, dim, ne); + const auto C = Reshape(coeff.Read(), coeff_vdim, q1d, q1d, ne); + auto DE = Reshape(pa_data.Write(), q1d, q1d, pa_size, + vdim * (matrix_coeff ? dim : 1), ne); + mfem::forall_2D(ne, q1d, q1d, [=] MFEM_HOST_DEVICE(int e) + { + MFEM_FOREACH_THREAD(qy, y, q1d) + { + MFEM_FOREACH_THREAD(qx, x, q1d) + { + const real_t J11 = J(qx, qy, 0, 0, e); + const real_t J21 = J(qx, qy, 1, 0, e); + const real_t J12 = J(qx, qy, 0, 1, e); + const real_t J22 = J(qx, qy, 1, 1, e); + const real_t w_detJ = W(qx, qy) / ((J11*J22)-(J21*J12)); + const real_t D0 = w_detJ * (J12*J12 + J22*J22); + const real_t D1 = -w_detJ * (J12*J11 + J22*J21); + const real_t D2 = w_detJ * (J11*J11 + J21*J21); + const int map[4] = {0, 2, 1, 3}; + + for (int i = 0; i < (matrix_coeff ? cvdim : nc); ++i) + { + const auto k = matrix_coeff ? map[i] : (vector_coeff ? i : 0); + const auto Cc = C(k, qx, qy, e); + DE(qx, qy, 0, i, e) = D0 * Cc; + DE(qx, qy, 1, i, e) = D1 * Cc; + DE(qx, qy, 2, i, e) = D1 * Cc; + DE(qx, qy, 3, i, e) = D2 * Cc; + } + } + } + }); + } + else if (dim == 3 && sdim == 3) + { + const int nc = vdim, cvdim = coeff_vdim; + const auto W = Reshape(ir->GetWeights().Read(), q1d, q1d, q1d); + const auto J = Reshape(geom->J.Read(), q1d, q1d, q1d, sdim, dim, ne); + const auto C = Reshape(coeff.Read(), coeff_vdim, q1d, q1d, q1d, ne); + auto DE = Reshape(pa_data.Write(), q1d, q1d, q1d, pa_size, + vdim * (matrix_coeff ? dim : 1), ne); + + mfem::forall_3D(ne, q1d, q1d, q1d, [=] MFEM_HOST_DEVICE(int e) + { + MFEM_FOREACH_THREAD(qz, z, q1d) + { + MFEM_FOREACH_THREAD(qy, y, q1d) + { + MFEM_FOREACH_THREAD(qx, x, q1d) + { + const real_t J11 = J(qx, qy, qz, 0, 0, e); + const real_t J21 = J(qx, qy, qz, 1, 0, e); + const real_t J31 = J(qx, qy, qz, 2, 0, e); + const real_t J12 = J(qx, qy, qz, 0, 1, e); + const real_t J22 = J(qx, qy, qz, 1, 1, e); + const real_t J32 = J(qx, qy, qz, 2, 1, e); + const real_t J13 = J(qx, qy, qz, 0, 2, e); + const real_t J23 = J(qx, qy, qz, 1, 2, e); + const real_t J33 = J(qx, qy, qz, 2, 2, e); + const real_t detJ = J11 * (J22 * J33 - J32 * J23) - + J21 * (J12 * J33 - J32 * J13) + + J31 * (J12 * J23 - J22 * J13); + const real_t c_detJ = W(qx, qy, qz) / detJ; + // adj(J) + const real_t A11 = (J22 * J33) - (J23 * J32); + const real_t A12 = (J32 * J13) - (J12 * J33); + const real_t A13 = (J12 * J23) - (J22 * J13); + const real_t A21 = (J31 * J23) - (J21 * J33); + const real_t A22 = (J11 * J33) - (J13 * J31); + const real_t A23 = (J21 * J13) - (J11 * J23); + const real_t A31 = (J21 * J32) - (J31 * J22); + const real_t A32 = (J31 * J12) - (J11 * J32); + const real_t A33 = (J11 * J22) - (J12 * J21); + // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T + const real_t D11 = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1 + const real_t D21 = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1 + const real_t D31 = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1 + const real_t D22 = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2 + const real_t D32 = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2 + const real_t D33 = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3 + const int map[9] = {0, 3, 6, 1, 4, 7, 2, 5, 8}; + + for (int i = 0; i < (matrix_coeff ? cvdim : nc); ++i) + { + const auto k = matrix_coeff ? map[i] : vector_coeff ? i : 0; + const auto Ck = C(k, qx, qy, qz, e); + DE(qx, qy, qz, 0, i, e) = D11 * Ck; + DE(qx, qy, qz, 1, i, e) = D21 * Ck; + DE(qx, qy, qz, 2, i, e) = D31 * Ck; + DE(qx, qy, qz, 3, i, e) = D22 * Ck; + DE(qx, qy, qz, 4, i, e) = D32 * Ck; + DE(qx, qy, qz, 5, i, e) = D33 * Ck; + } + } + } } }); } else { - PAVectorDiffusionSetup(dim, quad1D, ne, w, j, coeff, d); + MFEM_ABORT("Unknown VectorDiffusionIntegrator::AssemblePA kernel for" + << " dim:" << dim << ", vdim:" << vdim << ", sdim:" << sdim); } } +// PA Diffusion Apply kernel +void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const +{ + // Use CEED backend if available + if (DeviceCanUseCeed()) { return ceedOp->AddMult(x, y); } + + // Add the VectorDiffusionAddMultPA specializations + static const auto vector_diffusion_kernel_specializations = + ( + // 2D, SDIM = 2 + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 2,2>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 3,3>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 4,4>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 5,5>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 6,6>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 7,7>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 8,8>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,2, 9,9>::Add(), + // 2D, SDIM = 3 + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,3, 2,2>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,3, 3,3>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,3, 4,4>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<2,3, 5,5>::Add(), + // 3D, SDIM = 3 + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 2,2>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 2,3>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 3,4>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 4,5>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 4,6>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 5,6>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 5,8>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 6,7>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 7,8>::Add(), + VectorDiffusionIntegrator::ApplyPAKernels::Specialization<3,3, 8,9>::Add(), + true); + MFEM_CONTRACT_VAR(vector_diffusion_kernel_specializations); + + ApplyPAKernels::Run(dim, sdim, dofs1D, quad1D, + ne, coeff_vdim, maps->B, maps->G, pa_data, x, y, + sdim, dofs1D, quad1D); + +} + template static void PAVectorDiffusionDiagonal2D(const int NE, const Array &b, @@ -235,12 +338,15 @@ static void PAVectorDiffusionDiagonal2D(const int NE, const int Q1D = T_Q1D ? T_Q1D : q1d; MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto G = Reshape(g.Read(), Q1D, D1D); + + const auto B = Reshape(b.Read(), Q1D, D1D); + const auto G = Reshape(g.Read(), Q1D, D1D); // note the different shape for D, this is a (symmetric) matrix so we only // store necessary entries - auto D = Reshape(d.Read(), Q1D*Q1D, 3, NE); + MFEM_VERIFY(d.Size() == Q1D*Q1D*4*2*NE, ""); + const auto D = Reshape(d.Read(), Q1D*Q1D, /*3*/4, 2, NE); auto Y = Reshape(y.ReadWrite(), D1D, D1D, 2, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) { const int D1D = T_D1D ? T_D1D : d1d; @@ -261,9 +367,9 @@ static void PAVectorDiffusionDiagonal2D(const int NE, for (int qy = 0; qy < Q1D; ++qy) { const int q = qx + qy * Q1D; - const real_t D0 = D(q,0,e); - const real_t D1 = D(q,1,e); - const real_t D2 = D(q,2,e); + const real_t D0 = D(q,0,0,e); + const real_t D1 = D(q,1,0,e); + const real_t D2 = D(q,3/*2*/,0,e); // size from 3 (symmetric) to 4 (dims x dims) QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D0; QD1[qx][dy] += B(qy, dy) * G(qy, dy) * D1; QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D2; @@ -307,7 +413,8 @@ static void PAVectorDiffusionDiagonal3D(const int NE, MFEM_VERIFY(Q1D <= max_q1d, ""); auto B = Reshape(b.Read(), Q1D, D1D); auto G = Reshape(g.Read(), Q1D, D1D); - auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 6, NE); + MFEM_VERIFY(d.Size() == Q1D*Q1D*Q1D*9*3*NE, ""); + auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 9/*PA_SIZE:dims*dims*/, 3/*VDIM*/, NE); auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, 3, NE); mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) { @@ -335,7 +442,8 @@ static void PAVectorDiffusionDiagonal3D(const int NE, const int k = j >= i ? 3 - (3-i)*(2-i)/2 + j: 3 - (3-j)*(2-j)/2 + i; - const real_t O = Q(q,k,e); + // using 6 symmetric values + const real_t O = Q(q,k,0,e); const real_t Bz = B(qz,dz); const real_t Gz = G(qz,dz); const real_t L = i==2 ? Gz : Bz; @@ -419,328 +527,14 @@ void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag) } else { + MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported."); PAVectorDiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, maps->G, pa_data, diag); } } -// PA Diffusion Apply 2D kernel -template static -void PAVectorDiffusionApply2D(const int NE, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &d_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0, - const int vdim = 0) -{ - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - const int VDIM = T_VDIM ? T_VDIM : vdim; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto G = Reshape(g.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto Gt = Reshape(gt.Read(), D1D, Q1D); - auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE); - auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - const int VDIM = T_VDIM ? T_VDIM : vdim; - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - - real_t grad[max_Q1D][max_Q1D][2]; - for (int c = 0; c < VDIM; c++) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - grad[qy][qx][0] = 0.0; - grad[qy][qx][1] = 0.0; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - real_t gradX[max_Q1D][2]; - for (int qx = 0; qx < Q1D; ++qx) - { - gradX[qx][0] = 0.0; - gradX[qx][1] = 0.0; - } - for (int dx = 0; dx < D1D; ++dx) - { - const real_t s = x(dx,dy,c,e); - for (int qx = 0; qx < Q1D; ++qx) - { - gradX[qx][0] += s * B(qx,dx); - gradX[qx][1] += s * G(qx,dx); - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t wy = B(qy,dy); - const real_t wDy = G(qy,dy); - for (int qx = 0; qx < Q1D; ++qx) - { - grad[qy][qx][0] += gradX[qx][1] * wy; - grad[qy][qx][1] += gradX[qx][0] * wDy; - } - } - } - // Calculate Dxy, xDy in plane - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - const int q = qx + qy * Q1D; - const real_t O11 = D(q,0,e); - const real_t O12 = D(q,1,e); - const real_t O22 = D(q,2,e); - const real_t gradX = grad[qy][qx][0]; - const real_t gradY = grad[qy][qx][1]; - grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY); - grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY); - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - real_t gradX[max_D1D][2]; - for (int dx = 0; dx < D1D; ++dx) - { - gradX[dx][0] = 0.0; - gradX[dx][1] = 0.0; - } - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t gX = grad[qy][qx][0]; - const real_t gY = grad[qy][qx][1]; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t wx = Bt(dx,qx); - const real_t wDx = Gt(dx,qx); - gradX[dx][0] += gX * wDx; - gradX[dx][1] += gY * wx; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - const real_t wy = Bt(dy,qy); - const real_t wDy = Gt(dy,qy); - for (int dx = 0; dx < D1D; ++dx) - { - y(dx,dy,c,e) += ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy)); - } - } - } - } - }); -} - -// PA Diffusion Apply 3D kernel -template static -void PAVectorDiffusionApply3D(const int NE, - const Array &b, - const Array &g, - const Array &bt, - const Array >, - const Vector &op_, - const Vector &x_, - Vector &y_, - int d1d = 0, int q1d = 0) -{ - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int VDIM = 3; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(b.Read(), Q1D, D1D); - auto G = Reshape(g.Read(), Q1D, D1D); - auto Bt = Reshape(bt.Read(), D1D, Q1D); - auto Gt = Reshape(gt.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D*Q1D*Q1D, 6, NE); - auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - for (int c = 0; c < VDIM; ++ c) - { - real_t grad[max_Q1D][max_Q1D][max_Q1D][3]; - for (int qz = 0; qz < Q1D; ++qz) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - grad[qz][qy][qx][0] = 0.0; - grad[qz][qy][qx][1] = 0.0; - grad[qz][qy][qx][2] = 0.0; - } - } - } - for (int dz = 0; dz < D1D; ++dz) - { - real_t gradXY[max_Q1D][max_Q1D][3]; - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - gradXY[qy][qx][0] = 0.0; - gradXY[qy][qx][1] = 0.0; - gradXY[qy][qx][2] = 0.0; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - real_t gradX[max_Q1D][2]; - for (int qx = 0; qx < Q1D; ++qx) - { - gradX[qx][0] = 0.0; - gradX[qx][1] = 0.0; - } - for (int dx = 0; dx < D1D; ++dx) - { - const real_t s = x(dx,dy,dz,c,e); - for (int qx = 0; qx < Q1D; ++qx) - { - gradX[qx][0] += s * B(qx,dx); - gradX[qx][1] += s * G(qx,dx); - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t wy = B(qy,dy); - const real_t wDy = G(qy,dy); - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t wx = gradX[qx][0]; - const real_t wDx = gradX[qx][1]; - gradXY[qy][qx][0] += wDx * wy; - gradXY[qy][qx][1] += wx * wDy; - gradXY[qy][qx][2] += wx * wy; - } - } - } - for (int qz = 0; qz < Q1D; ++qz) - { - const real_t wz = B(qz,dz); - const real_t wDz = G(qz,dz); - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz; - grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz; - grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz; - } - } - } - } - // Calculate Dxyz, xDyz, xyDz in plane - for (int qz = 0; qz < Q1D; ++qz) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - const int q = qx + (qy + qz * Q1D) * Q1D; - const real_t O11 = op(q,0,e); - const real_t O12 = op(q,1,e); - const real_t O13 = op(q,2,e); - const real_t O22 = op(q,3,e); - const real_t O23 = op(q,4,e); - const real_t O33 = op(q,5,e); - const real_t gradX = grad[qz][qy][qx][0]; - const real_t gradY = grad[qz][qy][qx][1]; - const real_t gradZ = grad[qz][qy][qx][2]; - grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ); - grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ); - grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ); - } - } - } - for (int qz = 0; qz < Q1D; ++qz) - { - real_t gradXY[max_D1D][max_D1D][3]; - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - gradXY[dy][dx][0] = 0; - gradXY[dy][dx][1] = 0; - gradXY[dy][dx][2] = 0; - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - real_t gradX[max_D1D][3]; - for (int dx = 0; dx < D1D; ++dx) - { - gradX[dx][0] = 0; - gradX[dx][1] = 0; - gradX[dx][2] = 0; - } - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t gX = grad[qz][qy][qx][0]; - const real_t gY = grad[qz][qy][qx][1]; - const real_t gZ = grad[qz][qy][qx][2]; - for (int dx = 0; dx < D1D; ++dx) - { - const real_t wx = Bt(dx,qx); - const real_t wDx = Gt(dx,qx); - gradX[dx][0] += gX * wDx; - gradX[dx][1] += gY * wx; - gradX[dx][2] += gZ * wx; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - const real_t wy = Bt(dy,qy); - const real_t wDy = Gt(dy,qy); - for (int dx = 0; dx < D1D; ++dx) - { - gradXY[dy][dx][0] += gradX[dx][0] * wy; - gradXY[dy][dx][1] += gradX[dx][1] * wDy; - gradXY[dy][dx][2] += gradX[dx][2] * wy; - } - } - } - for (int dz = 0; dz < D1D; ++dz) - { - const real_t wz = Bt(dz,qz); - const real_t wDz = Gt(dz,qz); - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - y(dx,dy,dz,c,e) += - ((gradXY[dy][dx][0] * wz) + - (gradXY[dy][dx][1] * wz) + - (gradXY[dy][dx][2] * wDz)); - } - } - } - } - } - }); -} - +/* // PA Diffusion Apply kernel void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const { @@ -757,27 +551,30 @@ void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const const Array &Bt = maps->Bt; const Array &Gt = maps->Gt; const Vector &D = pa_data; - - if (dim == 2 && sdim == 3) - { - switch ((dofs1D << 4 ) | quad1D) - { - case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y); - case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y); - case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y); - case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y); - default: - return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); - } - } - if (dim == 2 && sdim == 2) - { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); } - - if (dim == 3 && sdim == 3) - { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); } - - MFEM_ABORT("Unknown kernel."); + ApplyPAKernels::Run(dim, sdim, D1D, Q1D, ne, B, G, Bt, Gt, D, x, y, D1D, + Q1D, sdim); } } +/// \cond DO_NOT_DOCUMENT + +VectorDiffusionIntegrator::ApplyKernelType +VectorDiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int, int) +{ + if (DIM == 2) { return internal::PAVectorDiffusionApply2D; } + else if (DIM == 3) { return internal::PAVectorDiffusionApply3D; } + else { MFEM_ABORT(""); } +} + +VectorDiffusionIntegrator::Kernels::Kernels() +{ + VectorDiffusionIntegrator::AddSpecialization<2, 3, 2, 2>(); + VectorDiffusionIntegrator::AddSpecialization<2, 3, 3, 3>(); + VectorDiffusionIntegrator::AddSpecialization<2, 3, 4, 4>(); + VectorDiffusionIntegrator::AddSpecialization<2, 3, 5, 5>(); +} + +/// \endcond DO_NOT_DOCUMENT +*/ + } // namespace mfem diff --git a/fem/integ/bilininteg_vecdiffusion_pa.hpp b/fem/integ/bilininteg_vecdiffusion_pa.hpp new file mode 100644 index 0000000000..8fc388e6c0 --- /dev/null +++ b/fem/integ/bilininteg_vecdiffusion_pa.hpp @@ -0,0 +1,202 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. +#pragma once + +#include "../../config/config.hpp" +#include "../../general/array.hpp" +#include "../../general/forall.hpp" +#include "../../linalg/dtensor.hpp" +#include "../../linalg/vector.hpp" +#include "../bilininteg.hpp" +#include "../kernels.hpp" + +using mfem::kernels::internal::SetMaxOf; + +namespace mfem +{ + +/// \cond DO_NOT_DOCUMENT + +namespace internal +{ + +template +void SmemPAVectorDiffusionApply2D(const int NE, + const int coeff_vdim, + const Array &b, + const Array &g, + const Vector &d, + const Vector &x, + Vector &y, + const int sdim = 0, + const int d1d = 0, + const int q1d = 0) +{ + static constexpr int DIM = 2; + const int SDIM = T_SDIM ? T_SDIM : sdim; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + + const int PA_SIZE = DIM*DIM; + const bool matrix_coeff = coeff_vdim == DIM*DIM; + + const auto B = b.Read(), G = g.Read(); + const auto DE = Reshape(d.Read(), Q1D, Q1D, PA_SIZE, + SDIM * (matrix_coeff ? SDIM : 1), NE); + const auto XE = Reshape(x.Read(), D1D, D1D, SDIM, NE); + auto YE = Reshape(y.ReadWrite(), D1D, D1D, SDIM, NE); + + mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e) + { + constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D; + constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D; + + MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1], smem[MQ1][MQ1]; + kernels::internal::vd_regs2d_t<3, DIM, MQ1> r0, r1; + kernels::internal::LoadMatrix(D1D, Q1D, B, sB); + kernels::internal::LoadMatrix(D1D, Q1D, G, sG); + + for (int i = 0; i < SDIM; i++) + { + for (int j = 0; j < (matrix_coeff ? SDIM : 1); j++) + { + kernels::internal::LoadDofs2d(e, D1D, i, XE, r0); + kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, r0, r1, i); + MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D) + { + MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D) + { + const real_t gradX = r1[i][0][qy][qx]; + const real_t gradY = r1[i][1][qy][qx]; + const int k = matrix_coeff ? (j + i * SDIM) : i; + const real_t O11 = DE(qx,qy,0,k,e), O12 = DE(qx,qy,1,k,e); + const real_t O21 = DE(qx,qy,2,k,e), O22 = DE(qx,qy,3,k,e); + r0[i][0][qy][qx] = (O11 * gradX) + (O12 * gradY); + r0[i][1][qy][qx] = (O21 * gradX) + (O22 * gradY); + } // qx + } // qy + MFEM_SYNC_THREAD; + kernels::internal::GradTranspose2d(D1D, Q1D, smem, sB, sG, r0, r1, i); + const int ij = matrix_coeff ? j : i; + kernels::internal::WriteDofs2d(e, D1D, i, ij, r1, YE); + } // j + } // i + }); +} + +template +void SmemPAVectorDiffusionApply3D(const int NE, + const int coeff_vdim, + const Array &b, + const Array &g, + const Vector &d, + const Vector &x, + Vector &y, + const int sdim = 0, + const int d1d = 0, + const int q1d = 0) +{ + + static constexpr int DIM = 3; + const int SDIM = T_SDIM ? T_SDIM : sdim; + MFEM_VERIFY(SDIM == 3, "SDIM must be 3"); + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + + const int PA_SIZE = DIM*DIM; + const bool matrix_coeff = coeff_vdim == DIM*DIM; + + const auto B = b.Read(), G = g.Read(); + const auto DE = Reshape(d.Read(), Q1D, Q1D, Q1D, PA_SIZE, + SDIM * (matrix_coeff ? SDIM : 1), NE); + const auto XE = Reshape(x.Read(), D1D, D1D, D1D, SDIM, NE); + auto YE = Reshape(y.ReadWrite(), D1D, D1D, D1D, SDIM, NE); + + mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e) + { + constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D; + constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D; + + MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1], smem[MQ1][MQ1]; + kernels::internal::vd_regs3d_t<3, DIM, MQ1> r0, r1; + kernels::internal::LoadMatrix(D1D, Q1D, B, sB); + kernels::internal::LoadMatrix(D1D, Q1D, G, sG); + + for (int i = 0; i < SDIM; i++) + { + for (int j = 0; j < (matrix_coeff ? SDIM : 1); j++) + { + kernels::internal::LoadDofs3d(e, D1D, i, XE, r0); + kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, r0, r1, i); + for (int qz = 0; qz < Q1D; qz++) + { + MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D) + { + MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D) + { + const real_t gradX = r1[i][0][qz][qy][qx]; + const real_t gradY = r1[i][1][qz][qy][qx]; + const real_t gradZ = r1[i][2][qz][qy][qx]; + const int k = matrix_coeff ? (j + i * SDIM) : i; + const real_t O11 = DE(qx,qy,qz,0,k,e), O12 = DE(qx,qy,qz,1,k,e), + O13 = DE(qx,qy,qz,2,k,e); + const real_t O22 = DE(qx,qy,qz,3,k,e), O23 = DE(qx,qy,qz,4,k,e); + const real_t O33 = DE(qx,qy,qz,5,k,e); + r0[i][0][qz][qy][qx] = (O11*gradX)+(O12*gradY)+(O13*gradZ); + r0[i][1][qz][qy][qx] = (O12*gradX)+(O22*gradY)+(O23*gradZ); + r0[i][2][qz][qy][qx] = (O13*gradX)+(O23*gradY)+(O33*gradZ); + } // qx + } // qy + } // qz + MFEM_SYNC_THREAD; + kernels::internal::GradTranspose3d(D1D, Q1D, smem, sB, sG, r0, r1, i); + const int ij = matrix_coeff ? j : i; + kernels::internal::WriteDofs3d(e, D1D, i, ij, r1, YE); + } // j + } // i + }); +} + +} // namespace internal + +template +VectorDiffusionIntegrator::ApplyKernelType +VectorDiffusionIntegrator::ApplyPAKernels::Kernel() +{ + if (DIM == 2) + { + return internal::SmemPAVectorDiffusionApply2D; + } + else if (DIM == 3) + { + return internal::SmemPAVectorDiffusionApply3D; + } + else { MFEM_ABORT("Unsupported kernel"); } +} + +inline VectorDiffusionIntegrator::ApplyKernelType +VectorDiffusionIntegrator::ApplyPAKernels::Fallback(int dim, int sdim, + int d1d, int q1d) +{ + if (dim == 2) + { + return internal::SmemPAVectorDiffusionApply2D; + } + else if (dim == 3) + { + return internal::SmemPAVectorDiffusionApply3D; + } + else { MFEM_ABORT("Unsupported kernel"); } +} + +/// \endcond DO_NOT_DOCUMENT + +} // namespace mfem diff --git a/fem/integ/bilininteg_vecmass_pa.cpp b/fem/integ/bilininteg_vecmass_pa.cpp index 8021e9095b..9dad5c471e 100644 --- a/fem/integ/bilininteg_vecmass_pa.cpp +++ b/fem/integ/bilininteg_vecmass_pa.cpp @@ -9,120 +9,218 @@ // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. -#include "../../general/forall.hpp" #include "../bilininteg.hpp" -#include "../gridfunc.hpp" +#include "../../general/forall.hpp" #include "../ceed/integrators/mass/mass.hpp" +#include "./bilininteg_vecmass_pa.hpp" // IWYU pragma: keep + namespace mfem { void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes) { - // Assuming the same element type Mesh *mesh = fes.GetMesh(); const FiniteElement &el = *fes.GetTypicalFE(); - ElementTransformation *T = mesh->GetTypicalElementTransformation(); - const IntegrationRule *ir - = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T); + ElementTransformation &Trans = *mesh->GetTypicalElementTransformation(); + const auto *ir = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, Trans); + if (DeviceCanUseCeed()) { delete ceedOp; - const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 || - fes.IsVariableOrder(); - if (mixed) - { - ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q); - } - else - { - ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q); - } + const bool mixed = + mesh->GetNumGeometries(mesh->Dimension()) > 1 || fes.IsVariableOrder(); + if (mixed) { ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q); } + else { ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q); } return; } + + // If vdim is not set, set it to the space dimension + vdim = (vdim == -1) ? Trans.GetSpaceDim() : vdim; + MFEM_VERIFY(vdim == fes.GetVDim(), "vdim != fes.GetVDim()"); + MFEM_VERIFY(vdim == mesh->Dimension(), "vdim != dim"); + + const MemoryType mt = pa_mt == MemoryType::DEFAULT + ? Device::GetDeviceMemoryType() + : pa_mt; + + ne = mesh->GetNE(); dim = mesh->Dimension(); - ne = fes.GetMesh()->GetNE(); - nq = ir->GetNPoints(); - geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES | - GeometricFactors::JACOBIANS); + const int nq = ir->GetNPoints(); + const int sdim = mesh->SpaceDimension(); + geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt); maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR); dofs1D = maps->ndof; quad1D = maps->nqpt; - pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType()); - real_t coeff = 1.0; + const int q1d = quad1D; + + if (!(dim == 2 || dim == 3)) { MFEM_ABORT("Dimension not supported."); } + + QuadratureSpace qs(*mesh, *ir); + CoefficientVector coeff(qs); + if (Q) { - ConstantCoefficient *cQ = dynamic_cast(Q); - MFEM_VERIFY(cQ != NULL, "Only ConstantCoefficient is supported."); - coeff = cQ->constant; + coeff.Project(*Q); } - if (!(dim == 2 || dim == 3)) + else if (VQ) { - MFEM_ABORT("Dimension not supported."); + coeff.Project(*VQ); + MFEM_VERIFY(VQ->GetVDim() == vdim, "VQ vdim vs. vdim error"); } + else if (MQ) + { + coeff.ProjectTranspose(*MQ); + MFEM_VERIFY(MQ->GetVDim() == vdim, "MQ dimension vs. vdim error"); + MFEM_VERIFY(coeff.Size() == (vdim*vdim) * ne * nq, "MQ size error"); + } + else { coeff.SetConstant(1.0); } + + coeff_vdim = coeff.GetVDim(); + const bool const_coeff = coeff_vdim == 1; + const bool vector_coeff = coeff_vdim == vdim; + const bool matrix_coeff = coeff_vdim == vdim * vdim; + MFEM_VERIFY(const_coeff + vector_coeff + matrix_coeff == 1, ""); + + pa_data.SetSize(coeff_vdim * nq * ne, mt); + + const auto w_r = ir->GetWeights().Read(); + if (dim == 2) { - const real_t constant = coeff; - const int NE = ne; - const int NQ = nq; - auto w = ir->GetWeights().Read(); - auto J = Reshape(geom->J.Read(), NQ,2,2,NE); - auto v = Reshape(pa_data.Write(), NQ, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + const auto W = Reshape(w_r, q1d, q1d); + const auto C = Reshape(coeff.Read(), coeff_vdim, q1d, q1d, ne); + const auto J = Reshape(geom->J.Read(), q1d, q1d, sdim, dim, ne); + auto D = Reshape(pa_data.Write(), q1d, q1d, coeff_vdim, ne); + + mfem::forall_2D(ne, q1d, q1d, [=] MFEM_HOST_DEVICE(int e) { - for (int q = 0; q < NQ; ++q) + MFEM_FOREACH_THREAD(qy, y, q1d) { - const real_t J11 = J(q,0,0,e); - const real_t J12 = J(q,1,0,e); - const real_t J21 = J(q,0,1,e); - const real_t J22 = J(q,1,1,e); - const real_t detJ = (J11*J22)-(J21*J12); - v(q,e) = w[q] * constant * detJ; + MFEM_FOREACH_THREAD(qx, x, q1d) + { + const real_t J11 = J(qx, qy, 0, 0, e), J12 = J(qx, qy, 1, 0, e); + const real_t J21 = J(qx, qy, 0, 1, e), J22 = J(qx, qy, 1, 1, e); + const real_t detJ = (J11 * J22) - (J21 * J12); + const real_t w_det = W(qx, qy) * detJ; + D(qx, qy, 0, e) = C(0, qx, qy, e) * w_det; + if (const_coeff) { continue; } + D(qx, qy, 1, e) = C(1, qx, qy, e) * w_det; + if (vector_coeff) { continue; } + assert(matrix_coeff); + D(qx, qy, 2, e) = C(2, qx, qy, e) * w_det; + D(qx, qy, 3, e) = C(3, qx, qy, e) * w_det; + } } }); } - if (dim == 3) + else if (dim == 3) { - const real_t constant = coeff; - const int NE = ne; - const int NQ = nq; - auto W = ir->GetWeights().Read(); - auto J = Reshape(geom->J.Read(), NQ,3,3,NE); - auto v = Reshape(pa_data.Write(), NQ,NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + const auto W = Reshape(w_r, q1d, q1d, q1d); + const auto C = Reshape(coeff.Read(), coeff_vdim, q1d, q1d, q1d, ne); + const auto J = Reshape(geom->J.Read(), q1d, q1d, q1d, sdim, dim, ne); + auto D = Reshape(pa_data.Write(), q1d, q1d, q1d, coeff_vdim, ne); + + mfem::forall_3D(ne, q1d, q1d, q1d, [=] MFEM_HOST_DEVICE(int e) { - for (int q = 0; q < NQ; ++q) + MFEM_FOREACH_THREAD(qz, z, q1d) { - const real_t J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e); - const real_t J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e); - const real_t J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e); - const real_t detJ = J11 * (J22 * J33 - J32 * J23) - - J21 * (J12 * J33 - J32 * J13) + - J31 * (J12 * J23 - J22 * J13); - v(q,e) = W[q] * constant * detJ; + MFEM_FOREACH_THREAD(qy, y, q1d) + { + MFEM_FOREACH_THREAD(qx, x, q1d) + { + const real_t J11 = J(qx, qy, qz, 0, 0, e), + J12 = J(qx, qy, qz, 0, 1, e), + J13 = J(qx, qy, qz, 0, 2, e); + const real_t J21 = J(qx, qy, qz, 1, 0, e), + J22 = J(qx, qy, qz, 1, 1, e), + J23 = J(qx, qy, qz, 1, 2, e); + const real_t J31 = J(qx, qy, qz, 2, 0, e), + J32 = J(qx, qy, qz, 2, 1, e), + J33 = J(qx, qy, qz, 2, 2, e); + const real_t detJ = J11 * (J22 * J33 - J32 * J23) - + J21 * (J12 * J33 - J32 * J13) + + J31 * (J12 * J23 - J22 * J13); + const real_t w_det = W(qx, qy, qz) * detJ; + D(qx, qy, qz, 0, e) = C(0, qx, qy, qz, e) * w_det; + if (const_coeff) { continue; } + D(qx, qy, qz, 1, e) = C(1, qx, qy, qz, e) * w_det; + D(qx, qy, qz, 2, e) = C(2, qx, qy, qz, e) * w_det; + if (vector_coeff) { continue; } + D(qx, qy, qz, 3, e) = C(3, qx, qy, qz, e) * w_det; + D(qx, qy, qz, 4, e) = C(4, qx, qy, qz, e) * w_det; + D(qx, qy, qz, 5, e) = C(5, qx, qy, qz, e) * w_det; + D(qx, qy, qz, 6, e) = C(6, qx, qy, qz, e) * w_det; + D(qx, qy, qz, 7, e) = C(7, qx, qy, qz, e) * w_det; + D(qx, qy, qz, 8, e) = C(8, qx, qy, qz, e) * w_det; + } + } } }); } + else + { + MFEM_ABORT("Unknown VectorMassIntegrator::AssemblePA kernel for" + << " dim:" << dim << ", vdim:" << vdim << ", sdim:" << sdim); + } } -template -static void PAVectorMassAssembleDiagonal2D(const int NE, - const Array &B_, - const Array &Bt_, - const Vector &op_, - Vector &diag_, - const int d1d = 0, - const int q1d = 0) +void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const { + // Use CEED backend if available + if (DeviceCanUseCeed()) { return ceedOp->AddMult(x, y); } + + // Add the VectorMassAddMultPA specializations + static const auto vector_mass_kernel_specializations = + ( // 2D + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 2,2>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 3,3>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 3,4>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 4,4>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 4,6>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 5,5>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 6,6>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 7,7>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 8,8>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<2, 9,9>::Add(), + // 3D + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 2,2>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 2,3>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 3,4>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 3,5>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 4,5>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 4,6>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 4,8>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 5,6>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 5,8>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 6,7>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 7,8>::Add(), + VectorMassIntegrator::VectorMassAddMultPA::Specialization<3, 8,9>::Add(), + true); + MFEM_CONTRACT_VAR(vector_mass_kernel_specializations); + + VectorMassAddMultPA::Run(dim, dofs1D, quad1D, + ne, coeff_vdim, maps->B, pa_data, x, y, + dofs1D, quad1D); + +} + +template +static void PAVectorMassAssembleDiagonal2D(const int NE, + const Array &b, + const Vector &pa_data, Vector &diag, + const int d1d = 0, const int q1d = 0) +{ + constexpr int VDIM = 2; const int D1D = T_D1D ? T_D1D : d1d; const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int VDIM = 2; MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(B_.Read(), Q1D, D1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, NE); - auto y = Reshape(diag_.ReadWrite(), D1D, D1D, VDIM, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + const auto B = Reshape(b.Read(), Q1D, D1D); + const auto D = Reshape(pa_data.Read(), Q1D, Q1D, NE); + auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE); + + mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e) { const int D1D = T_D1D ? T_D1D : d1d; const int Q1D = T_Q1D ? T_Q1D : q1d; @@ -137,7 +235,7 @@ static void PAVectorMassAssembleDiagonal2D(const int NE, temp[qx][dy] = 0.0; for (int qy = 0; qy < Q1D; ++qy) { - temp[qx][dy] += B(qy, dy) * B(qy, dy) * op(qx, qy, e); + temp[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e); } } } @@ -150,33 +248,31 @@ static void PAVectorMassAssembleDiagonal2D(const int NE, { temp1 += B(qx, dx) * B(qx, dx) * temp[qx][dy]; } - y(dx, dy, 0, e) = temp1; - y(dx, dy, 1, e) = temp1; + Y(dx, dy, 0, e) = temp1; + Y(dx, dy, 1, e) = temp1; } } }); } -template +template static void PAVectorMassAssembleDiagonal3D(const int NE, const Array &B_, - const Array &Bt_, - const Vector &op_, - Vector &diag_, - const int d1d = 0, - const int q1d = 0) + const Vector &pa_data, Vector &diag, + const int d1d = 0, const int q1d = 0) { + constexpr int VDIM = 3; const int D1D = T_D1D ? T_D1D : d1d; const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int VDIM = 3; MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(B_.Read(), Q1D, D1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE); - auto y = Reshape(diag_.ReadWrite(), D1D, D1D, D1D, VDIM, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) + const auto B = Reshape(B_.Read(), Q1D, D1D); + MFEM_VERIFY(pa_data.Size() == Q1D * Q1D * Q1D * NE, "pa_data size error"); + const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, NE); + auto Y = Reshape(diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE); + mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e) { - const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround + const int D1D = T_D1D ? T_D1D : d1d; const int Q1D = T_Q1D ? T_Q1D : q1d; // the following variables are evaluated at compile time constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; @@ -192,7 +288,8 @@ static void PAVectorMassAssembleDiagonal3D(const int NE, temp[qx][qy][dz] = 0.0; for (int qz = 0; qz < Q1D; ++qz) { - temp[qx][qy][dz] += B(qz, dz) * B(qz, dz) * op(qx, qy, qz, e); + temp[qx][qy][dz] += + B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e); } } } @@ -207,7 +304,8 @@ static void PAVectorMassAssembleDiagonal3D(const int NE, temp2[qx][dy][dz] = 0.0; for (int qy = 0; qy < Q1D; ++qy) { - temp2[qx][dy][dz] += B(qy, dy) * B(qy, dy) * temp[qx][qy][dz]; + temp2[qx][dy][dz] += + B(qy, dy) * B(qy, dy) * temp[qx][qy][dz]; } } } @@ -221,323 +319,42 @@ static void PAVectorMassAssembleDiagonal3D(const int NE, real_t temp3 = 0.0; for (int qx = 0; qx < Q1D; ++qx) { - temp3 += B(qx, dx) * B(qx, dx) - * temp2[qx][dy][dz]; + temp3 += B(qx, dx) * B(qx, dx) * temp2[qx][dy][dz]; } - y(dx, dy, dz, 0, e) = temp3; - y(dx, dy, dz, 1, e) = temp3; - y(dx, dy, dz, 2, e) = temp3; + Y(dx, dy, dz, 0, e) = temp3; + Y(dx, dy, dz, 1, e) = temp3; + Y(dx, dy, dz, 2, e) = temp3; } } } }); } -static void PAVectorMassAssembleDiagonal(const int dim, - const int D1D, - const int Q1D, - const int NE, +static void PAVectorMassAssembleDiagonal(const int dim, const int D1D, + const int Q1D, const int NE, const Array &B, - const Array &Bt, - const Vector &op, - Vector &y) + const Vector &pa_data, + Vector &diag) { if (dim == 2) { - return PAVectorMassAssembleDiagonal2D(NE, B, Bt, op, y, D1D, Q1D); + return PAVectorMassAssembleDiagonal2D(NE, B, pa_data, diag, D1D, Q1D); } else if (dim == 3) { - return PAVectorMassAssembleDiagonal3D(NE, B, Bt, op, y, D1D, Q1D); + return PAVectorMassAssembleDiagonal3D(NE, B, pa_data, diag, D1D, Q1D); } MFEM_ABORT("Dimension not implemented."); } void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag) { - if (DeviceCanUseCeed()) - { - ceedOp->GetDiagonal(diag); - } + if (DeviceCanUseCeed()) { ceedOp->GetDiagonal(diag); } else { - PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, - maps->B, maps->Bt, - pa_data, diag); - } -} - -template -static void PAVectorMassApply2D(const int NE, - const Array &B_, - const Array &Bt_, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int VDIM = 2; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(B_.Read(), Q1D, D1D); - auto Bt = Reshape(Bt_.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, NE); - auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround - const int Q1D = T_Q1D ? T_Q1D : q1d; - // the following variables are evaluated at compile time - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - real_t sol_xy[max_Q1D][max_Q1D]; - for (int c = 0; c < VDIM; ++c) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xy[qy][qx] = 0.0; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - real_t sol_x[max_Q1D]; - for (int qy = 0; qy < Q1D; ++qy) - { - sol_x[qy] = 0.0; - } - for (int dx = 0; dx < D1D; ++dx) - { - const real_t s = x(dx,dy,c,e); - for (int qx = 0; qx < Q1D; ++qx) - { - sol_x[qx] += B(qx,dx)* s; - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t d2q = B(qy,dy); - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xy[qy][qx] += d2q * sol_x[qx]; - } - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xy[qy][qx] *= op(qx,qy,e); - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - real_t sol_x[max_D1D]; - for (int dx = 0; dx < D1D; ++dx) - { - sol_x[dx] = 0.0; - } - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t s = sol_xy[qy][qx]; - for (int dx = 0; dx < D1D; ++dx) - { - sol_x[dx] += Bt(dx,qx) * s; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - const real_t q2d = Bt(dy,qy); - for (int dx = 0; dx < D1D; ++dx) - { - y(dx,dy,c,e) += q2d * sol_x[dx]; - } - } - } - } - }); -} - -template -static void PAVectorMassApply3D(const int NE, - const Array &B_, - const Array &Bt_, - const Vector &op_, - const Vector &x_, - Vector &y_, - const int d1d = 0, - const int q1d = 0) -{ - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int VDIM = 3; - MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, ""); - MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, ""); - auto B = Reshape(B_.Read(), Q1D, D1D); - auto Bt = Reshape(Bt_.Read(), D1D, Q1D); - auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE); - auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE); - auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE); - mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) - { - const int D1D = T_D1D ? T_D1D : d1d; - const int Q1D = T_Q1D ? T_Q1D : q1d; - constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - real_t sol_xyz[max_Q1D][max_Q1D][max_Q1D]; - for (int c = 0; c < VDIM; ++ c) - { - for (int qz = 0; qz < Q1D; ++qz) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xyz[qz][qy][qx] = 0.0; - } - } - } - for (int dz = 0; dz < D1D; ++dz) - { - real_t sol_xy[max_Q1D][max_Q1D]; - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xy[qy][qx] = 0.0; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - real_t sol_x[max_Q1D]; - for (int qx = 0; qx < Q1D; ++qx) - { - sol_x[qx] = 0; - } - for (int dx = 0; dx < D1D; ++dx) - { - const real_t s = x(dx,dy,dz,c,e); - for (int qx = 0; qx < Q1D; ++qx) - { - sol_x[qx] += B(qx,dx) * s; - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - const real_t wy = B(qy,dy); - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xy[qy][qx] += wy * sol_x[qx]; - } - } - } - for (int qz = 0; qz < Q1D; ++qz) - { - const real_t wz = B(qz,dz); - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx]; - } - } - } - } - for (int qz = 0; qz < Q1D; ++qz) - { - for (int qy = 0; qy < Q1D; ++qy) - { - for (int qx = 0; qx < Q1D; ++qx) - { - sol_xyz[qz][qy][qx] *= op(qx,qy,qz,e); - } - } - } - for (int qz = 0; qz < Q1D; ++qz) - { - real_t sol_xy[max_D1D][max_D1D]; - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - sol_xy[dy][dx] = 0; - } - } - for (int qy = 0; qy < Q1D; ++qy) - { - real_t sol_x[max_D1D]; - for (int dx = 0; dx < D1D; ++dx) - { - sol_x[dx] = 0; - } - for (int qx = 0; qx < Q1D; ++qx) - { - const real_t s = sol_xyz[qz][qy][qx]; - for (int dx = 0; dx < D1D; ++dx) - { - sol_x[dx] += Bt(dx,qx) * s; - } - } - for (int dy = 0; dy < D1D; ++dy) - { - const real_t wy = Bt(dy,qy); - for (int dx = 0; dx < D1D; ++dx) - { - sol_xy[dy][dx] += wy * sol_x[dx]; - } - } - } - for (int dz = 0; dz < D1D; ++dz) - { - const real_t wz = Bt(dz,qz); - for (int dy = 0; dy < D1D; ++dy) - { - for (int dx = 0; dx < D1D; ++dx) - { - y(dx,dy,dz,c,e) += wz * sol_xy[dy][dx]; - } - } - } - } - } - }); -} - -static void PAVectorMassApply(const int dim, - const int D1D, - const int Q1D, - const int NE, - const Array &B, - const Array &Bt, - const Vector &op, - const Vector &x, - Vector &y) -{ - if (dim == 2) - { - return PAVectorMassApply2D(NE, B, Bt, op, x, y, D1D, Q1D); - } - if (dim == 3) - { - return PAVectorMassApply3D(NE, B, Bt, op, x, y, D1D, Q1D); - } - MFEM_ABORT("Unknown kernel."); -} - -void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const -{ - if (DeviceCanUseCeed()) - { - ceedOp->AddMult(x, y); - } - else - { - PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y); + MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1"); + MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported"); + PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag); } } diff --git a/fem/integ/bilininteg_vecmass_pa.hpp b/fem/integ/bilininteg_vecmass_pa.hpp new file mode 100644 index 0000000000..09fc4bf755 --- /dev/null +++ b/fem/integ/bilininteg_vecmass_pa.hpp @@ -0,0 +1,212 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. +#pragma once + +#include "../../config/config.hpp" +#include "../../general/array.hpp" +#include "../../general/forall.hpp" +#include "../../linalg/dtensor.hpp" +#include "../../linalg/vector.hpp" +#include "../bilininteg.hpp" +#include "../kernels.hpp" + +using mfem::kernels::internal::SetMaxOf; + +namespace mfem +{ + +/// \cond DO_NOT_DOCUMENT + +namespace internal +{ + +template +void SmemPAVectorMassApply2D(const int NE, + const int coeff_vdim, + const Array &b, + const Vector &d, + const Vector &x, + Vector &y, + const int d1d = 0, + const int q1d = 0) +{ + static constexpr int DIM = 2, VDIM = 2; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + + const bool const_coeff = coeff_vdim == 1; + const bool vector_coeff = coeff_vdim == DIM; + const bool matrix_coeff = coeff_vdim == DIM*DIM; + + const auto B = b.Read(); + const auto D = Reshape(d.Read(), Q1D, Q1D, coeff_vdim, NE); + const auto X = Reshape(x.Read(), D1D, D1D, VDIM, NE); + auto Y = Reshape(y.ReadWrite(), D1D, D1D, VDIM, NE); + + mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e) + { + constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D; + constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D; + + MFEM_SHARED real_t sB[MD1][MQ1], smem[MQ1][MQ1]; + kernels::internal::v_regs2d_t r0, r1; + kernels::internal::LoadMatrix(D1D, Q1D, B, sB); + kernels::internal::LoadDofs2d(e, D1D, X, r0); + kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); + + MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D) + { + MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D) + { + const real_t Qx = r1[0][qy][qx]; + const real_t Qy = r1[1][qy][qx]; + const real_t D0 = D(qx, qy, 0, e); + + if (const_coeff) + { + r0[0][qy][qx] = D0 * Qx; + r0[1][qy][qx] = D0 * Qy; + } + if (vector_coeff) + { + const real_t D1 = D(qx, qy, 1, e); + r0[0][qy][qx] = D0 * Qx; + r0[1][qy][qx] = D1 * Qy; + } + if (matrix_coeff) + { + const real_t D1 = D(qx, qy, 1, e); + const real_t D2 = D(qx, qy, 2, e); + const real_t D3 = D(qx, qy, 3, e); + r0[0][qy][qx] = D0 * Qx + D1 * Qy; + r0[1][qy][qx] = D2 * Qx + D3 * Qy; + } + } + } + kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r0, r1); + kernels::internal::WriteDofs2d(e, D1D, r1, Y); + }); +} + +template +void SmemPAVectorMassApply3D(const int NE, + const int coeff_vdim, + const Array &b, + const Vector &d, + const Vector &x, + Vector &y, + const int d1d = 0, + const int q1d = 0) +{ + static constexpr int VDIM = 3; + const int D1D = T_D1D ? T_D1D : d1d; + const int Q1D = T_Q1D ? T_Q1D : q1d; + + const bool const_coeff = coeff_vdim == 1; + const bool vector_coeff = coeff_vdim == VDIM; + const bool matrix_coeff = coeff_vdim == VDIM*VDIM; + + const auto B = b.Read(); + const auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, coeff_vdim, NE); + const auto X = Reshape(x.Read(), D1D, D1D, D1D, VDIM, NE); + auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, VDIM, NE); + + mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e) + { + constexpr int MD1 = T_D1D > 0 ? SetMaxOf(T_D1D) : DofQuadLimits::MAX_T1D; + constexpr int MQ1 = T_Q1D > 0 ? SetMaxOf(T_Q1D) : DofQuadLimits::MAX_T1D; + + MFEM_SHARED real_t sB[MD1][MQ1], smem[MQ1][MQ1]; + kernels::internal::v_regs3d_t r0, r1; + kernels::internal::LoadMatrix(D1D, Q1D, B, sB); + kernels::internal::LoadDofs3d(e, D1D, X, r0); + kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); + + for (int qz = 0; qz < Q1D; qz++) + { + MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D) + { + MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D) + { + const real_t Qx = r1[0][qz][qy][qx]; + const real_t Qy = r1[1][qz][qy][qx]; + const real_t Qz = r1[2][qz][qy][qx]; + const real_t D0 = D(qx, qy, qz, 0, e); + if (const_coeff) + { + r0[0][qz][qy][qx] = D0 * Qx; + r0[1][qz][qy][qx] = D0 * Qy; + r0[2][qz][qy][qx] = D0 * Qz; + } + if (vector_coeff) + { + const real_t D1 = D(qx, qy, qz, 1, e); + const real_t D2 = D(qx, qy, qz, 2, e); + r0[0][qz][qy][qx] = D0 * Qx; + r0[1][qz][qy][qx] = D1 * Qy; + r0[2][qz][qy][qx] = D2 * Qz; + } + if (matrix_coeff) + { + const real_t D1 = D(qx, qy, qz, 1, e); + const real_t D2 = D(qx, qy, qz, 2, e); + const real_t D3 = D(qx, qy, qz, 3, e); + const real_t D4 = D(qx, qy, qz, 4, e); + const real_t D5 = D(qx, qy, qz, 5, e); + const real_t D6 = D(qx, qy, qz, 6, e); + const real_t D7 = D(qx, qy, qz, 7, e); + const real_t D8 = D(qx, qy, qz, 8, e); + r0[0][qz][qy][qx] = D0 * Qx + D1 * Qy + D2 * Qz; + r0[1][qz][qy][qx] = D3 * Qx + D4 * Qy + D5 * Qz; + r0[2][qz][qy][qx] = D6 * Qx + D7 * Qy + D8 * Qz; + } + } + } + } + kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r0, r1); + kernels::internal::WriteDofs3d(e, D1D, r1, Y); + }); +} + +} // namespace internal + +template +VectorMassIntegrator::VectorMassAddMultPAType +VectorMassIntegrator::VectorMassAddMultPA::Kernel() +{ + if (DIM == 2) + { + return internal::SmemPAVectorMassApply2D; + } + else if (DIM == 3) + { + return internal::SmemPAVectorMassApply3D; + } + else { MFEM_ABORT("Unsupported kernel"); } +} + +inline VectorMassIntegrator::VectorMassAddMultPAType +VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d) +{ + if (dim == 2) + { + return internal::SmemPAVectorMassApply2D; + } + else if (dim == 3) + { + return internal::SmemPAVectorMassApply3D; + } + else { MFEM_ABORT("Unsupported kernel"); } +} + +/// \endcond DO_NOT_DOCUMENT + +} // namespace mfem diff --git a/fem/integ/lininteg_domain.cpp b/fem/integ/lininteg_domain.cpp index 64dfe961d0..da202181d1 100644 --- a/fem/integ/lininteg_domain.cpp +++ b/fem/integ/lininteg_domain.cpp @@ -9,183 +9,19 @@ // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. -#include "../../general/forall.hpp" #include "../../fem/kernels.hpp" +#include "../../general/forall.hpp" #include "../fem.hpp" +#include "lininteg_domain_kernels.hpp" + +/// \cond DO_NOT_DOCUMENT + namespace mfem { - -template -static void DLFEvalAssemble2D(const int vdim, const int ne, const int d, - const int q, - const int map_type, const int *markers, const real_t *b, - const real_t *detj, const real_t *weights, - const Vector &coeff, real_t *y) -{ - const auto F = coeff.Read(); - const auto M = Reshape(markers, ne); - const auto B = Reshape(b, q, d); - const auto DETJ = Reshape(detj, q, q, ne); - const auto W = Reshape(weights, q, q); - const bool cst = coeff.Size() == vdim; - const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne); - auto Y = Reshape(y, d,d, vdim, ne); - - mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE (int e) - { - if (M(e) == 0) { return; } // ignore - - constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - - MFEM_SHARED real_t sBt[Q*D]; - MFEM_SHARED real_t sQQ[Q*Q]; - MFEM_SHARED real_t sQD[Q*D]; - - const DeviceMatrix Bt(sBt, d, q); - kernels::internal::LoadB(d, q, B, sBt); - - const DeviceMatrix QQ(sQQ, q, q); - const DeviceMatrix QD(sQD, q, d); - - for (int c = 0; c < vdim; ++c) - { - const real_t cst_val = C(c,0,0,0); - MFEM_FOREACH_THREAD(x,x,q) - { - MFEM_FOREACH_THREAD(y,y,q) - { - const real_t detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,e) : 1.0; - const real_t coeff_val = cst ? cst_val : C(c,x,y,e); - QQ(y,x) = W(x,y) * coeff_val * detJ; - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(qy,y,q) - { - MFEM_FOREACH_THREAD(dx,x,d) - { - real_t u = 0.0; - for (int qx = 0; qx < q; ++qx) { u += QQ(qy,qx) * Bt(dx,qx); } - QD(qy,dx) = u; - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(dy,y,d) - { - MFEM_FOREACH_THREAD(dx,x,d) - { - real_t u = 0.0; - for (int qy = 0; qy < q; ++qy) { u += QD(qy,dx) * Bt(dy,qy); } - Y(dx,dy,c,e) += u; - } - } - MFEM_SYNC_THREAD; - } - }); -} - -template -static void DLFEvalAssemble3D(const int vdim, const int ne, const int d, - const int q, - const int map_type, const int *markers, const real_t *b, - const real_t *detj, const real_t *weights, - const Vector &coeff, real_t *y) -{ - const auto F = coeff.Read(); - const auto M = Reshape(markers, ne); - const auto B = Reshape(b, q,d); - const auto DETJ = Reshape(detj, q, q, q, ne); - const auto W = Reshape(weights, q,q,q); - const bool cst_coeff = coeff.Size() == vdim; - const auto C = cst_coeff ? Reshape(F,vdim,1,1,1,1):Reshape(F,vdim,q,q,q,ne); - - auto Y = Reshape(y, d,d,d, vdim, ne); - - mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE (int e) - { - if (M(e) == 0) { return; } // ignore - - constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; - constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; - constexpr int MQD = (Q >= D) ? Q : D; - - real_t u[D]; - - MFEM_SHARED real_t sBt[Q*D]; - const DeviceMatrix Bt(sBt, d,q); - kernels::internal::LoadB(d,q,B,sBt); - - MFEM_SHARED real_t sQQQ[MQD*MQD*MQD]; - const DeviceCube QQQ(sQQQ, MQD, MQD, MQD); - - for (int c = 0; c < vdim; ++c) - { - const real_t cst_val = C(c,0,0,0,0); - MFEM_FOREACH_THREAD(x,x,q) - { - MFEM_FOREACH_THREAD(y,y,q) - { - for (int z = 0; z < q; ++z) - { - const real_t detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,z,e) : 1.0; - const real_t coeff_val = cst_coeff ? cst_val : C(c,x,y,z,e); - QQQ(z,y,x) = W(x,y,z) * coeff_val * detJ; - } - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(qx,x,q) - { - MFEM_FOREACH_THREAD(qy,y,q) - { - for (int dz = 0; dz < d; ++dz) { u[dz] = 0.0; } - for (int qz = 0; qz < q; ++qz) - { - const real_t ZYX = QQQ(qz,qy,qx); - for (int dz = 0; dz < d; ++dz) { u[dz] += ZYX * Bt(dz,qz); } - } - for (int dz = 0; dz < d; ++dz) { QQQ(dz,qy,qx) = u[dz]; } - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(dz,y,d) - { - MFEM_FOREACH_THREAD(qx,x,q) - { - for (int dy = 0; dy < d; ++dy) { u[dy] = 0.0; } - for (int qy = 0; qy < q; ++qy) - { - const real_t zYX = QQQ(dz,qy,qx); - for (int dy = 0; dy < d; ++dy) { u[dy] += zYX * Bt(dy,qy); } - } - for (int dy = 0; dy < d; ++dy) { QQQ(dz,dy,qx) = u[dy]; } - } - } - MFEM_SYNC_THREAD; - MFEM_FOREACH_THREAD(dz,y,d) - { - MFEM_FOREACH_THREAD(dy,x,d) - { - for (int dx = 0; dx < d; ++dx) { u[dx] = 0.0; } - for (int qx = 0; qx < q; ++qx) - { - const real_t zyX = QQQ(dz,dy,qx); - for (int dx = 0; dx < d; ++dx) { u[dx] += zyX * Bt(dx,qx); } - } - for (int dx = 0; dx < d; ++dx) { Y(dx,dy,dz,c,e) += u[dx]; } - } - } - MFEM_SYNC_THREAD; - } - }); -} - static void DLFEvalAssemble(const FiniteElementSpace &fes, const IntegrationRule *ir, - const Array &markers, - const Vector &coeff, + const Array &markers, const Vector &coeff, Vector &y) { Mesh *mesh = fes.GetMesh(); @@ -197,50 +33,20 @@ static void DLFEvalAssemble(const FiniteElementSpace &fes, constexpr int flags = GeometricFactors::DETERMINANTS; const GeometricFactors *geom = mesh->GetGeometricFactors(*ir, flags, mt); const int map_type = fes.GetTypicalFE()->GetMapType(); - decltype(&DLFEvalAssemble2D<>) ker = - dim == 2 ? DLFEvalAssemble2D<> : DLFEvalAssemble3D<>; - - if (dim==2) - { - if (d==1 && q==1) { ker=DLFEvalAssemble2D<1,1>; } - if (d==2 && q==2) { ker=DLFEvalAssemble2D<2,2>; } - if (d==3 && q==3) { ker=DLFEvalAssemble2D<3,3>; } - if (d==4 && q==4) { ker=DLFEvalAssemble2D<4,4>; } - if (d==5 && q==5) { ker=DLFEvalAssemble2D<5,5>; } - if (d==2 && q==3) { ker=DLFEvalAssemble2D<2,3>; } - if (d==3 && q==4) { ker=DLFEvalAssemble2D<3,4>; } - if (d==4 && q==5) { ker=DLFEvalAssemble2D<4,5>; } - if (d==5 && q==6) { ker=DLFEvalAssemble2D<5,6>; } - } - - if (dim==3) - { - if (d==1 && q==1) { ker=DLFEvalAssemble3D<1,1>; } - if (d==2 && q==2) { ker=DLFEvalAssemble3D<2,2>; } - if (d==3 && q==3) { ker=DLFEvalAssemble3D<3,3>; } - if (d==4 && q==4) { ker=DLFEvalAssemble3D<4,4>; } - if (d==5 && q==5) { ker=DLFEvalAssemble3D<5,5>; } - if (d==2 && q==3) { ker=DLFEvalAssemble3D<2,3>; } - if (d==3 && q==4) { ker=DLFEvalAssemble3D<3,4>; } - if (d==4 && q==5) { ker=DLFEvalAssemble3D<4,5>; } - if (d==5 && q==6) { ker=DLFEvalAssemble3D<5,6>; } - } - - MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q); const int vdim = fes.GetVDim(); const int ne = fes.GetMesh()->GetNE(); - const int *M = markers.Read(); const real_t *B = maps.B.Read(); + const int *M = markers.Read(); const real_t *detJ = geom->detJ.Read(); const real_t *W = ir->GetWeights().Read(); real_t *Y = y.ReadWrite(); - ker(vdim, ne, d, q, map_type, M, B, detJ, W, coeff, Y); + DomainLFIntegrator::AssembleKernels::Run(dim, d, q, vdim, ne, d, q, map_type, + M, B, detJ, W, coeff, Y); } void DomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes, - const Array &markers, - Vector &b) + const Array &markers, Vector &b) { const FiniteElement &fe = *fes.GetTypicalFE(); const int qorder = oa * fe.GetOrder() + ob; @@ -266,4 +72,49 @@ void VectorDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes, DLFEvalAssemble(fes, ir, markers, coeff, b); } +DomainLFIntegrator::AssembleKernelType +DomainLFIntegrator::AssembleKernels::Fallback(int DIM, int, int) +{ + switch (DIM) + { + case 1: + return DLFEvalAssemble1D<0, 0>; + case 2: + return DLFEvalAssemble2D<0, 0>; + case 3: + return DLFEvalAssemble3D<0, 0>; + } + MFEM_ABORT(""); +} + +DomainLFIntegrator::Kernels::Kernels() +{ + // 2D + // Q = P+1 + DomainLFIntegrator::AddSpecialization<2, 1, 1>(); + DomainLFIntegrator::AddSpecialization<2, 2, 2>(); + DomainLFIntegrator::AddSpecialization<2, 3, 3>(); + DomainLFIntegrator::AddSpecialization<2, 4, 4>(); + DomainLFIntegrator::AddSpecialization<2, 5, 5>(); + // Q = P+2 + DomainLFIntegrator::AddSpecialization<2, 2, 3>(); + DomainLFIntegrator::AddSpecialization<2, 3, 4>(); + DomainLFIntegrator::AddSpecialization<2, 4, 5>(); + DomainLFIntegrator::AddSpecialization<2, 5, 6>(); + // 3D + // Q = P+1 + DomainLFIntegrator::AddSpecialization<3, 1, 1>(); + DomainLFIntegrator::AddSpecialization<3, 2, 2>(); + DomainLFIntegrator::AddSpecialization<3, 3, 3>(); + DomainLFIntegrator::AddSpecialization<3, 4, 4>(); + DomainLFIntegrator::AddSpecialization<3, 5, 5>(); + // Q = P+2 + DomainLFIntegrator::AddSpecialization<3, 2, 3>(); + DomainLFIntegrator::AddSpecialization<3, 3, 4>(); + DomainLFIntegrator::AddSpecialization<3, 4, 5>(); + DomainLFIntegrator::AddSpecialization<3, 5, 6>(); +} + +/// \endcond DO_NOT_DOCUMENT + } // namespace mfem diff --git a/fem/integ/lininteg_domain_kernels.hpp b/fem/integ/lininteg_domain_kernels.hpp new file mode 100644 index 0000000000..ec28ed1a7e --- /dev/null +++ b/fem/integ/lininteg_domain_kernels.hpp @@ -0,0 +1,318 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_LININTEG_DOMAIN_KERNELS_HPP +#define MFEM_LININTEG_DOMAIN_KERNELS_HPP + +#include "../../fem/kernels.hpp" +#include "../../general/forall.hpp" +#include "../fem.hpp" + +/// \cond DO_NOT_DOCUMENT + +namespace mfem +{ + +template +static void DLFEvalAssemble1D(const int vdim, const int ne, const int d, + const int q, const int map_type, + const int *markers, const real_t *b, + const real_t *detj, const real_t *weights, + const Vector &coeff, real_t *y) +{ + { + constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + MFEM_VERIFY(q <= Q, ""); + MFEM_VERIFY(d <= D, ""); + } + + const auto F = coeff.Read(); + const auto B = Reshape(b, q, d); + const auto DETJ = Reshape(detj, q, ne); + const bool cst = coeff.Size() == vdim; + const auto C = cst ? Reshape(F, vdim, 1, 1) : Reshape(F, vdim, q, ne); + auto Y = Reshape(y, d, vdim, ne); + + mfem::forall_2D(ne, d, 1, [=] MFEM_HOST_DEVICE(int e) + { + if (markers[e] == 0) + { + return; + } // ignore + + constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + + MFEM_SHARED real_t sBt[Q * D]; + const DeviceMatrix Bt(sBt, d, q); + kernels::internal::LoadB(d, q, B, sBt); + + for (int c = 0; c < vdim; ++c) + { + const real_t cst_val = C(c, 0, 0); + MFEM_FOREACH_THREAD(dx, x, d) + { + real_t u = 0; + for (int qx = 0; qx < q; ++qx) + { + const real_t detJ = + (map_type == FiniteElement::VALUE) ? DETJ(qx, e) : 1.0; + const real_t coeff_val = cst ? cst_val : C(c, qx, e); + u += weights[qx] * coeff_val * detJ * Bt(dx, qx); + } + Y(dx, c, e) += u; + } + } + }); +} + +template +static void DLFEvalAssemble2D(const int vdim, const int ne, const int d, + const int q, const int map_type, + const int *markers, const real_t *b, + const real_t *detj, const real_t *weights, + const Vector &coeff, real_t *y) +{ + { + constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + MFEM_VERIFY(q <= Q, ""); + MFEM_VERIFY(d <= D, ""); + } + + const auto F = coeff.Read(); + const auto B = Reshape(b, q, d); + const auto DETJ = Reshape(detj, q, q, ne); + const auto W = Reshape(weights, q, q); + const bool cst = coeff.Size() == vdim; + const auto C = cst ? Reshape(F, vdim, 1, 1, 1) : Reshape(F, vdim, q, q, ne); + auto Y = Reshape(y, d, d, vdim, ne); + + mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE(int e) + { + if (markers[e] == 0) + { + return; + } // ignore + + constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + + MFEM_SHARED real_t sBt[Q * D]; + MFEM_SHARED real_t sQQ[Q * Q]; + MFEM_SHARED real_t sQD[Q * D]; + + const DeviceMatrix Bt(sBt, d, q); + kernels::internal::LoadB(d, q, B, sBt); + + const DeviceMatrix QQ(sQQ, q, q); + const DeviceMatrix QD(sQD, q, d); + + for (int c = 0; c < vdim; ++c) + { + const real_t cst_val = C(c, 0, 0, 0); + MFEM_FOREACH_THREAD(x, x, q) + { + MFEM_FOREACH_THREAD(y, y, q) + { + const real_t detJ = + (map_type == FiniteElement::VALUE) ? DETJ(x, y, e) : 1.0; + const real_t coeff_val = cst ? cst_val : C(c, x, y, e); + QQ(y, x) = W(x, y) * coeff_val * detJ; + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(qy, y, q) + { + MFEM_FOREACH_THREAD(dx, x, d) + { + real_t u = 0.0; + for (int qx = 0; qx < q; ++qx) + { + u += QQ(qy, qx) * Bt(dx, qx); + } + QD(qy, dx) = u; + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(dy, y, d) + { + MFEM_FOREACH_THREAD(dx, x, d) + { + real_t u = 0.0; + for (int qy = 0; qy < q; ++qy) + { + u += QD(qy, dx) * Bt(dy, qy); + } + Y(dx, dy, c, e) += u; + } + } + MFEM_SYNC_THREAD; + } + }); +} + +template +static void DLFEvalAssemble3D(const int vdim, const int ne, const int d, + const int q, const int map_type, + const int* markers, const real_t *b, + const real_t *detj, const real_t *weights, + const Vector &coeff, real_t *y) +{ + { + constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + MFEM_VERIFY(q <= Q, ""); + MFEM_VERIFY(d <= D, ""); + } + + const auto F = coeff.Read(); + const auto B = Reshape(b, q, d); + const auto DETJ = Reshape(detj, q, q, q, ne); + const auto W = Reshape(weights, q, q, q); + const bool cst_coeff = coeff.Size() == vdim; + const auto C = + cst_coeff ? Reshape(F, vdim, 1, 1, 1, 1) : Reshape(F, vdim, q, q, q, ne); + + auto Y = Reshape(y, d, d, d, vdim, ne); + + mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE(int e) + { + if (markers[e] == 0) + { + return; + } // ignore + + constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D; + constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D; + constexpr int MQD = (Q >= D) ? Q : D; + + real_t u[D]; + + MFEM_SHARED real_t sBt[Q * D]; + const DeviceMatrix Bt(sBt, d, q); + kernels::internal::LoadB(d, q, B, sBt); + + MFEM_SHARED real_t sQQQ[MQD * MQD * MQD]; + const DeviceCube QQQ(sQQQ, MQD, MQD, MQD); + + for (int c = 0; c < vdim; ++c) + { + const real_t cst_val = C(c, 0, 0, 0, 0); + MFEM_FOREACH_THREAD(x, x, q) + { + MFEM_FOREACH_THREAD(y, y, q) + { + for (int z = 0; z < q; ++z) + { + const real_t detJ = (map_type == FiniteElement::VALUE) + ? DETJ(x, y, z, e) + : 1.0; + const real_t coeff_val = + cst_coeff ? cst_val : C(c, x, y, z, e); + QQQ(z, y, x) = W(x, y, z) * coeff_val * detJ; + } + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(qx, x, q) + { + MFEM_FOREACH_THREAD(qy, y, q) + { + for (int dz = 0; dz < d; ++dz) + { + u[dz] = 0.0; + } + for (int qz = 0; qz < q; ++qz) + { + const real_t ZYX = QQQ(qz, qy, qx); + for (int dz = 0; dz < d; ++dz) + { + u[dz] += ZYX * Bt(dz, qz); + } + } + for (int dz = 0; dz < d; ++dz) + { + QQQ(dz, qy, qx) = u[dz]; + } + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(dz, y, d) + { + MFEM_FOREACH_THREAD(qx, x, q) + { + for (int dy = 0; dy < d; ++dy) + { + u[dy] = 0.0; + } + for (int qy = 0; qy < q; ++qy) + { + const real_t zYX = QQQ(dz, qy, qx); + for (int dy = 0; dy < d; ++dy) + { + u[dy] += zYX * Bt(dy, qy); + } + } + for (int dy = 0; dy < d; ++dy) + { + QQQ(dz, dy, qx) = u[dy]; + } + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD(dz, y, d) + { + MFEM_FOREACH_THREAD(dy, x, d) + { + for (int dx = 0; dx < d; ++dx) + { + u[dx] = 0.0; + } + for (int qx = 0; qx < q; ++qx) + { + const real_t zyX = QQQ(dz, dy, qx); + for (int dx = 0; dx < d; ++dx) + { + u[dx] += zyX * Bt(dx, qx); + } + } + for (int dx = 0; dx < d; ++dx) + { + Y(dx, dy, dz, c, e) += u[dx]; + } + } + } + MFEM_SYNC_THREAD; + } + }); +} + +template +DomainLFIntegrator::AssembleKernelType +DomainLFIntegrator::AssembleKernels::Kernel() +{ + switch (DIM) + { + case 1: + return DLFEvalAssemble1D; + case 2: + return DLFEvalAssemble2D; + case 3: + return DLFEvalAssemble3D; + } + MFEM_ABORT(""); +} +/// \endcond DO_NOT_DOCUMENT + +} // namespace mfem +#endif diff --git a/fem/intrules.cpp b/fem/intrules.cpp index 60bce30f68..a88ba2d0a0 100644 --- a/fem/intrules.cpp +++ b/fem/intrules.cpp @@ -947,6 +947,7 @@ int Quadrature1D::CheckOpen(int type) case OpenUniform: case ClosedUniform: case OpenHalfUniform: + case ClosedGL: return type; // all types can work as open default: return Invalid; diff --git a/fem/kernel_dispatch.hpp b/fem/kernel_dispatch.hpp index 92de1a4375..cdd16a3138 100644 --- a/fem/kernel_dispatch.hpp +++ b/fem/kernel_dispatch.hpp @@ -78,7 +78,7 @@ namespace mfem public: \ const char *kernel_name = MFEM_KERNEL_NAME(KernelName); \ using KernelSignature = KernelType; \ - template static MFEM_EXPORT KernelSignature Kernel(); \ + template static KernelSignature Kernel(); \ static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1); \ static MFEM_EXPORT KernelName &Get() { \ static KernelName table; \ diff --git a/fem/kernels.hpp b/fem/kernels.hpp index 6247293917..e60691cb42 100644 --- a/fem/kernels.hpp +++ b/fem/kernels.hpp @@ -14,6 +14,7 @@ #include "../config/config.hpp" #include "../linalg/dtensor.hpp" +#include "../linalg/tensor.hpp" namespace mfem { @@ -26,7 +27,704 @@ namespace kernels namespace internal { -/// Load B1d matrice into shared memory +// Types for tensors mapped to registers +// - N is the number of threads in each of the x and y dimensions +// - N should not be greater than 32, to have a maximum of 1024 threads +// On GPU, the last two dimensions are set to 0 to match a 2D tile of threads +#if ((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \ + (defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__))) +template +using s_regs2d_t = mfem::future::tensor; + +template +using v_regs2d_t = mfem::future::tensor; + +template +using vd_regs2d_t = mfem::future::tensor; + +template +using s_regs3d_t = mfem::future::tensor; + +template +using v_regs3d_t = mfem::future::tensor; + +template +using vd_regs3d_t = mfem::future::tensor; + +// on GPU, SetMaxOf is a no-op, for minimal register usage +constexpr int SetMaxOf(int n) { return n; } +#else +template +using s_regs2d_t = mfem::future::tensor; + +template +using v_regs2d_t = mfem::future::tensor; + +template +using vd_regs2d_t = mfem::future::tensor; + +template +using s_regs3d_t = mfem::future::tensor; + +template +using v_regs3d_t = mfem::future::tensor; + +template +using vd_regs3d_t = mfem::future::tensor; + +// on CPU, get next multiple of 4, allowing better alignments +template +constexpr int NextMultipleOf(int n) +{ + static_assert(N > 0 && (N & (N - 1)) == 0, "N must be a power of 2"); + return (n + (N - 1)) & ~(N - 1); +} +constexpr int SetMaxOf(int n) { return NextMultipleOf<4>(n); } +#endif // CUDA/HIP && DEVICE_COMPILE + +/// Load 2D matrix into shared memory +template +inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d, + const real_t *M, real_t (*N)[MQ1]) +{ + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d) + { + N[dy][qx] = M[dy * q1d + qx]; + } + } + MFEM_SYNC_THREAD; +} + +/// Load 2D input VDIM*DIM vector into given register tensor, specific component +template +inline MFEM_HOST_DEVICE void LoadDofs2d(const int e, const int d1d, const int c, + const DeviceTensor<4, const real_t> &X, + vd_regs2d_t &Y) +{ + for (int d = 0; d < DIM; d++) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y[c][d][dy][dx] = X(dx, dy, c, e); + } + } + } +} + +/// Load 2D input VDIM*DIM vector into given register tensor +template +inline MFEM_HOST_DEVICE void LoadDofs2d(const int e, const int d1d, + const DeviceTensor<4, const real_t> &X, + vd_regs2d_t &Y) +{ + for (int c = 0; c < VDIM; ++c) { LoadDofs2d(e, d1d, c, X, Y); } +} + +/// Load 2D input VDIM vector into given register tensor +template +inline MFEM_HOST_DEVICE void LoadDofs2d(const int e, const int d1d, + const DeviceTensor<4, const real_t> &X, + v_regs2d_t &Y) +{ + for (int c = 0; c < VDIM; ++c) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y[c][dy][dx] = X(dx, dy, c, e); + } + } + } +} + +/// Load 2D input scalar into given register tensor +template +inline MFEM_HOST_DEVICE void LoadDofs2d(const int e, const int d1d, + const DeviceTensor<3, const real_t> &X, + s_regs2d_t &Y) +{ + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y[dy][dx] = X(dx, dy, e); + } + } +} + +/// Write 2D vector into given device tensor, with read (i) write (j) indices +template +inline MFEM_HOST_DEVICE void WriteDofs2d(const int e, const int d1d, + const int i, const int j, + vd_regs2d_t &X, + const DeviceTensor<4, real_t> &Y) +{ + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + real_t y = 0.0; + for (int d = 0; d < DIM; d++) { y += X(i, d, dy, dx); } + Y(dx, dy, j, e) += y; + } + } +} + +/// Write 2D VDIM*DIM vector into given device tensor +template +inline MFEM_HOST_DEVICE void WriteDofs2d(const int e, const int d1d, + vd_regs2d_t &X, + const DeviceTensor<4, real_t> &Y) +{ + for (int c = 0; c < VDIM; ++c) { WriteDofs2d(e, d1d, c, c, X, Y); } +} + +/// Write 2D VDIM vector into given device tensor +template +inline MFEM_HOST_DEVICE void WriteDofs2d(const int e, const int d1d, + v_regs2d_t &X, + const DeviceTensor<4, real_t> &Y) +{ + for (int c = 0; c < VDIM; ++c) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y(dx, dy, c, e) += X(c, dy, dx); + } + } + } +} + +/// Load 3D input VDIM*DIM vector into given register tensor, specific component +template +inline MFEM_HOST_DEVICE void LoadDofs3d(const int e, const int d1d, const int c, + const DeviceTensor<5, const real_t> &X, + vd_regs3d_t &Y) +{ + for (int d = 0; d < DIM; d++) + { + for (int dz = 0; dz < d1d; ++dz) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y[c][d][dz][dy][dx] = X(dx, dy, dz, c, e); + } + } + } + } +} + +/// Load 3D input VDIM*DIM vector into given register tensor +template +inline MFEM_HOST_DEVICE void LoadDofs3d(const int e, const int d1d, + const DeviceTensor<5, const real_t> &X, + vd_regs3d_t &Y) +{ + for (int c = 0; c < VDIM; ++c) { LoadDofs3d(e, d1d, c, X, Y); } +} + +/// Load 3D input VDIM vector into given register tensor +template +inline MFEM_HOST_DEVICE void LoadDofs3d(const int e, const int d1d, + const DeviceTensor<5, const real_t> &X, + v_regs3d_t &Y) +{ + for (int c = 0; c < VDIM; ++c) + { + for (int dz = 0; dz < d1d; ++dz) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y[c][dz][dy][dx] = X(dx,dy,dz,c,e); + } + } + } + } +} + +/// Load 3D input scalar into given register tensor +template +inline MFEM_HOST_DEVICE void LoadDofs3d(const int e, const int d1d, + const DeviceTensor<4, const real_t> &X, + s_regs3d_t &Y) +{ + for (int dz = 0; dz < d1d; ++dz) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y[dz][dy][dx] = X(dx,dy,dz,e); + } + } + } +} + +/// Write 3D scalar into given device tensor, with read (i) write (j) indices +template +inline MFEM_HOST_DEVICE void WriteDofs3d(const int e, const int d1d, + const int i, const int j, + vd_regs3d_t &X, + const DeviceTensor<5, real_t> &Y) +{ + for (int dz = 0; dz < d1d; ++dz) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + real_t value = 0.0; + for (int d = 0; d < DIM; d++) { value += X(i, d, dz, dy, dx); } + Y(dx, dy, dz, j, e) += value; + } + } + } +} + +/// Write 3D VDIM*DIM vector into given device tensor +template +inline MFEM_HOST_DEVICE void WriteDofs3d(const int e, const int d1d, + vd_regs3d_t &X, + const DeviceTensor<5, real_t> &Y) +{ + for (int c = 0; c < VDIM; ++c) { WriteDofs3d(e, d1d, c, c, X, Y); } +} + +/// Write 3D VDIM vector into given device tensor +template +inline MFEM_HOST_DEVICE void WriteDofs3d(const int e, const int d1d, + v_regs3d_t &X, + const DeviceTensor<5, real_t> &Y) +{ + for (int c = 0; c < VDIM; ++c) + { + for (int dz = 0; dz < d1d; ++dz) + { + MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d) + { + Y(dx, dy, dz, c, e) += X(c, dz, dy, dx); + } + } + } + } +} + +/// 2D scalar contraction, X direction +template +inline MFEM_HOST_DEVICE void ContractX2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const s_regs2d_t &X, + s_regs2d_t &Y) +{ + MFEM_FOREACH_THREAD_DIRECT(y, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, (Transpose ? q1d : d1d)) + { + smem[y][x] = X[y][x]; + } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD_DIRECT(y, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, (Transpose ? d1d : q1d)) + { + real_t u = 0.0; + for (int k = 0; k < (Transpose ? q1d : d1d); ++k) + { + u += (Transpose ? B[x][k] : B[k][x]) * smem[y][k]; + } + Y[y][x] = u; + } + } + MFEM_SYNC_THREAD; +} + +/// 2D scalar contraction, Y direction +template +inline MFEM_HOST_DEVICE void ContractY2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const s_regs2d_t &X, + s_regs2d_t &Y) +{ + MFEM_FOREACH_THREAD_DIRECT(y, y, (Transpose ? q1d : d1d)) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, q1d) { smem[y][x] = X[y][x]; } + } + MFEM_SYNC_THREAD; + MFEM_FOREACH_THREAD_DIRECT(y, y, (Transpose ? d1d : q1d)) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, q1d) + { + real_t u = 0.0; + for (int k = 0; k < (Transpose ? q1d : d1d); ++k) + { + u += (Transpose ? B[y][k] : B[k][y]) * smem[k][x]; + } + Y[y][x] = u; + } + } + MFEM_SYNC_THREAD; +} + +/// 2D scalar copy +template +inline MFEM_HOST_DEVICE void Copy2d(const int q1d, + s_regs2d_t &X, + s_regs2d_t &Y) +{ + MFEM_FOREACH_THREAD_DIRECT(y, y, q1d) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, q1d) { Y[y][x] = X[y][x]; } + } +} + +/// 2D scalar contraction: X & Y directions, with additional copy +template +inline MFEM_HOST_DEVICE void Contract2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*Bx)[MQ1], + const real_t (*By)[MQ1], + s_regs2d_t &X, + s_regs2d_t &Y) +{ + if (!Transpose) + { + ContractX2d(d1d, q1d, smem, Bx, X, Y); + ContractY2d(d1d, q1d, smem, By, Y, X); + Copy2d(q1d, X, Y); + } + else + { + Copy2d(q1d, X, Y); + ContractY2d(d1d, q1d, smem, By, Y, X); + ContractX2d(d1d, q1d, smem, Bx, X, Y); + } +} + +/// 2D scalar evaluation +template +inline MFEM_HOST_DEVICE void Eval2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + s_regs2d_t &X, + s_regs2d_t &Y) +{ + Contract2d(d1d, q1d, smem, B, B, X, Y); +} + +/// 2D vector evaluation +template +inline MFEM_HOST_DEVICE void Eval2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + v_regs2d_t &X, + v_regs2d_t &Y) +{ + for (int c = 0; c < VDIM; c++) + { + Eval2d(d1d, q1d, smem, B, X[c], Y[c]); + } +} + +/// 2D vector transposed evaluation +template +inline MFEM_HOST_DEVICE void EvalTranspose2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + v_regs2d_t &X, + v_regs2d_t &Y) +{ + Eval2d(d1d, q1d, smem, B, X, Y); +} + +/// 2D vector gradient, with component +template +inline MFEM_HOST_DEVICE void Grad2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs2d_t &X, + vd_regs2d_t &Y, + const int c) +{ + + for (int d = 0; d < DIM; d++) + { + const real_t (*Bx)[MQ1] = (d == 0) ? G : B; + const real_t (*By)[MQ1] = (d == 1) ? G : B; + Contract2d(d1d, q1d, smem, Bx, By, X[c][d], Y[c][d]); + } + +} + +/// 2D vector gradient +template +inline MFEM_HOST_DEVICE void Grad2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs2d_t &X, + vd_regs2d_t &Y) +{ + for (int c = 0; c < VDIM; ++c) + { + Grad2d(d1d, q1d, smem, B, G, X, Y, c); + } +} + +/// 2D vector transposed gradient +template +inline MFEM_HOST_DEVICE void GradTranspose2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs2d_t &X, + vd_regs2d_t &Y) +{ + constexpr bool Transpose = true; + Grad2d(d1d, q1d, smem, B, G, X, Y); +} + +/// 2D scalar contraction, with component +template +inline MFEM_HOST_DEVICE void GradTranspose2d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs2d_t &X, + vd_regs2d_t &Y, + const int c) +{ + constexpr bool Transpose = true; + Grad2d(d1d, q1d, smem, B, G, X, Y, c); +} + +/// 3D scalar contraction, X direction +template +inline MFEM_HOST_DEVICE void ContractX3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const s_regs3d_t &X, + s_regs3d_t &Y) +{ + for (int z = 0; z < d1d; ++z) + { + MFEM_FOREACH_THREAD_DIRECT(y, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, (Transpose ? q1d : d1d)) + { + smem[y][x] = X[z][y][x]; + } + } + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD_DIRECT(y, y, d1d) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, (Transpose ? d1d : q1d)) + { + real_t u = 0.0; + for (int k = 0; k < (Transpose ? q1d : d1d); ++k) + { + u += (Transpose ? B[x][k] : B[k][x]) * smem[y][k]; + } + Y[z][y][x] = u; + } + } + MFEM_SYNC_THREAD; + } +} + +/// 3D scalar contraction, Y direction +template +inline MFEM_HOST_DEVICE void ContractY3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const s_regs3d_t &X, + s_regs3d_t &Y) +{ + for (int z = 0; z < d1d; ++z) + { + MFEM_FOREACH_THREAD_DIRECT(y, y, (Transpose ? q1d : d1d)) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, q1d) { smem[y][x] = X[z][y][x]; } + } + MFEM_SYNC_THREAD; + + MFEM_FOREACH_THREAD_DIRECT(y, y, (Transpose ? d1d : q1d)) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, q1d) + { + real_t u = 0.0; + for (int k = 0; k < (Transpose ? q1d : d1d); ++k) + { + u += (Transpose ? B[y][k] : B[k][y]) * smem[k][x]; + } + Y[z][y][x] = u; + } + } + MFEM_SYNC_THREAD; + } +} + +/// 3D scalar contraction, Z direction +template +inline MFEM_HOST_DEVICE void ContractZ3d(const int d1d, const int q1d, + const real_t (*B)[MQ1], + const s_regs3d_t &X, + s_regs3d_t &Y) +{ + for (int z = 0; z < (Transpose ? d1d : q1d); ++z) + { + MFEM_FOREACH_THREAD_DIRECT(y, y, q1d) + { + MFEM_FOREACH_THREAD_DIRECT(x, x, q1d) + { + real_t u = 0.0; + for (int k = 0; k < (Transpose ? q1d : d1d); ++k) + { + u += (Transpose ? B[z][k] : B[k][z]) * X[k][y][x]; + } + Y[z][y][x] = u; + } + } + } +} + +/// 3D scalar contraction: X, Y & Z directions +template +inline MFEM_HOST_DEVICE void Contract3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*Bx)[MQ1], + const real_t (*By)[MQ1], + const real_t (*Bz)[MQ1], + s_regs3d_t &X, + s_regs3d_t &Y) +{ + if (!Transpose) + { + ContractX3d(d1d, q1d, smem, Bx, X, Y); + ContractY3d(d1d, q1d, smem, By, Y, X); + ContractZ3d(d1d, q1d, Bz, X, Y); + } + else + { + ContractZ3d(d1d, q1d, Bz, X, Y); + ContractY3d(d1d, q1d, smem, By, Y, X); + ContractX3d(d1d, q1d, smem, Bx, X, Y); + } +} + +/// 3D scalar evaluation +template +inline MFEM_HOST_DEVICE void Eval3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + s_regs3d_t &X, + s_regs3d_t &Y) +{ + Contract3d(d1d, q1d, smem, B, B, B, X, Y); +} + +/// 3D vector evaluation +template +inline MFEM_HOST_DEVICE void Eval3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + v_regs3d_t &X, + v_regs3d_t &Y) +{ + for (int c = 0; c < VDIM; c++) + { + Eval3d(d1d, q1d, smem, B, X[c], Y[c]); + } +} + +/// 3D vector transposed evaluation +template +inline MFEM_HOST_DEVICE void EvalTranspose3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + v_regs3d_t &X, + v_regs3d_t &Y) +{ + Eval3d(d1d, q1d, smem, B, X, Y); +} + +/// 3D vector gradient, with component +template +inline MFEM_HOST_DEVICE void Grad3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs3d_t &X, + vd_regs3d_t &Y, + const int c) +{ + for (int d = 0; d < DIM; d++) + { + const real_t (*Bx)[MQ1] = (d == 0) ? G : B; + const real_t (*By)[MQ1] = (d == 1) ? G : B; + const real_t (*Bz)[MQ1] = (d == 2) ? G : B; + Contract3d(d1d, q1d, smem, Bx, By, Bz, X[c][d], Y[c][d]); + } +} + +/// 3D vector gradient +template +inline MFEM_HOST_DEVICE void Grad3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs3d_t &X, + vd_regs3d_t &Y) +{ + for (int c = 0; c < VDIM; c++) + { + Grad3d(d1d, q1d, smem, B, G, X, Y, c); + } +} + +/// 3D vector transposed gradient +template +inline MFEM_HOST_DEVICE void GradTranspose3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs3d_t &X, + vd_regs3d_t &Y) +{ + Grad3d(d1d, q1d, smem, B, G, X, Y); +} + +/// 3D vector transposed gradient, with component +template +inline MFEM_HOST_DEVICE void GradTranspose3d(const int d1d, const int q1d, + real_t (&smem)[MQ1][MQ1], + const real_t (*B)[MQ1], + const real_t (*G)[MQ1], + vd_regs3d_t &X, + vd_regs3d_t &Y, + const int c) +{ + Grad3d(d1d, q1d, smem, B, G, X, Y, c); +} + +/// Load B1d matrix into shared memory template MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D, const ConstDeviceMatrix &b, @@ -48,7 +746,7 @@ MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D, MFEM_SYNC_THREAD; } -/// Load Bt1d matrices into shared memory +/// Load Bt1d matrix into shared memory template MFEM_HOST_DEVICE inline void LoadBt(const int D1D, const int Q1D, const ConstDeviceMatrix &b, @@ -1551,7 +2249,7 @@ MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D, } } -} // namespace kernels::internal +} // namespace internal } // namespace kernels diff --git a/fem/linearform_ext.cpp b/fem/linearform_ext.cpp index 3cbbf3c1f6..23f3e2035d 100644 --- a/fem/linearform_ext.cpp +++ b/fem/linearform_ext.cpp @@ -51,7 +51,7 @@ void LinearFormExtension::Assemble() { // scan the attributes to set the markers to 0 or 1 const int NE = fes.GetNE(); - const auto attr = attributes.Read(); + const auto attr = attributes->Read(); const auto dimk = domain_integs_marker_k->Read(); auto markers_w = markers.Write(); mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e) @@ -93,13 +93,14 @@ void LinearFormExtension::Assemble() else { // scan the attributes to set the markers to 0 or 1 - const int NBE = bdr_attributes.Size(); - const auto attr = bdr_attributes.Read(); + const int NBE = bdr_face_attributes->Size(); + const auto attr = bdr_face_attributes->Read(); const auto attr_markers = boundary_integs_marker_k->Read(); auto markers_w = bdr_markers.Write(); - mfem::forall(NBE, [=] MFEM_HOST_DEVICE (int e) + mfem::forall(NBE, [=] MFEM_HOST_DEVICE(int e) { - markers_w[e] = attr_markers[attr[e]-1] == 1; + markers_w[e] = + attr[e] > 0 ? (attr_markers[attr[e] - 1] == 1) : false; }); } @@ -125,8 +126,7 @@ void LinearFormExtension::Update() //markers.UseDevice(true); // Gather the attributes on the host from all the elements - attributes.SetSize(NE); - for (int i = 0; i < NE; ++i) { attributes[i] = mesh.GetAttribute(i); } + attributes = &mesh.GetElementAttributes(); elem_restrict_lex = fes.GetElementRestriction(ordering); MFEM_VERIFY(elem_restrict_lex, "Element restriction not available"); @@ -136,34 +136,12 @@ void LinearFormExtension::Update() if (lf->boundary_integs.Size() > 0) { - const int nf_bdr = fes.GetNFbyType(FaceType::Boundary); + bdr_face_attributes = &mesh.GetBdrFaceAttributes(); + + const int nf_bdr = bdr_face_attributes->Size(); bdr_markers.SetSize(nf_bdr); // bdr_markers.UseDevice(true); - // The face restriction will give us "face E-vectors" on the boundary that - // are numbered in the order of the faces of mesh. This numbering will be - // different than the numbering of the boundary elements. We compute - // mappings so that the array `bdr_attributes[i]` gives the boundary - // attribute of the `i`th boundary face in the mesh face order. - std::unordered_map f_to_be; - for (int i = 0; i < mesh.GetNBE(); ++i) - { - const int f = mesh.GetBdrElementFaceIndex(i); - f_to_be[f] = i; - } - MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes"); - bdr_attributes.SetSize(nf_bdr); - int f_ind = 0; - for (int f = 0; f < mesh.GetNumFaces(); ++f) - { - if (f_to_be.find(f) != f_to_be.end()) - { - const int be = f_to_be[f]; - bdr_attributes[f_ind] = mesh.GetBdrAttribute(be); - ++f_ind; - } - } - bdr_restrict_lex = dynamic_cast( fes.GetFaceRestriction(ordering, FaceType::Boundary, diff --git a/fem/linearform_ext.hpp b/fem/linearform_ext.hpp index 4c5278c2e0..6d1827dc4c 100644 --- a/fem/linearform_ext.hpp +++ b/fem/linearform_ext.hpp @@ -25,7 +25,8 @@ class LinearForm; class LinearFormExtension { /// Attributes of all mesh elements. - Array attributes, bdr_attributes; + const Array *attributes; // Not owned + const Array *bdr_face_attributes; // Not owned /// Temporary markers for device kernels. Array markers, bdr_markers; diff --git a/fem/lininteg.cpp b/fem/lininteg.cpp index 61340ae34d..5d49f2b4db 100644 --- a/fem/lininteg.cpp +++ b/fem/lininteg.cpp @@ -35,6 +35,19 @@ void LinearFormIntegrator::AssembleRHSElementVect( mfem_error("LinearFormIntegrator::AssembleRHSElementVect(...)"); } +DomainLFIntegrator::DomainLFIntegrator(Coefficient &QF, int a, int b) + : DeltaLFIntegrator(QF), Q(QF), oa(a), ob(b) +{ + static Kernels kernels; +} + +DomainLFIntegrator::DomainLFIntegrator(Coefficient &QF, + const IntegrationRule *ir) + : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) +{ + static Kernels kernels; +} + void DomainLFIntegrator::AssembleRHSElementVect(const FiniteElement &el, ElementTransformation &Tr, Vector &elvect) @@ -266,6 +279,13 @@ void BoundaryTangentialLFIntegrator::AssembleRHSElementVect( } } +VectorDomainLFIntegrator::VectorDomainLFIntegrator(VectorCoefficient &QF, + const IntegrationRule *ir) + : DeltaLFIntegrator(QF, ir), Q(QF) +{ + static DomainLFIntegrator::Kernels kernels; +} + void VectorDomainLFIntegrator::AssembleRHSElementVect( const FiniteElement &el, ElementTransformation &Tr, Vector &elvect) { diff --git a/fem/lininteg.hpp b/fem/lininteg.hpp index 29777530dc..0bb52e68f6 100644 --- a/fem/lininteg.hpp +++ b/fem/lininteg.hpp @@ -18,6 +18,8 @@ #include #include "integrator.hpp" +#include "kernel_dispatch.hpp" + namespace mfem { @@ -109,14 +111,12 @@ class DomainLFIntegrator : public DeltaLFIntegrator int oa, ob; public: /// Constructs a domain integrator with a given Coefficient - DomainLFIntegrator(Coefficient &QF, int a = 2, int b = 0) - // the old default was a = 1, b = 1 - // for simple elliptic problems a = 2, b = -2 is OK - : DeltaLFIntegrator(QF), Q(QF), oa(a), ob(b) { } + /// the old default was a = 1, b = 1 + /// for simple elliptic problems a = 2, b = -2 is OK + DomainLFIntegrator(Coefficient &QF, int a = 2, int b = 0); /// Constructs a domain integrator with a given Coefficient - DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir) - : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) { } + DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir); bool SupportsDevice() const override { return true; } @@ -136,6 +136,22 @@ public: Vector &elvect) override; using LinearFormIntegrator::AssembleRHSElementVect; + + /// args: vdim, ne, d1d, q1d, map_type, markers, B, detJ, W, coeff, y + using AssembleKernelType = void (*)(const int, const int, const int, + const int, const int, const int *, + const real_t *, const real_t *, + const real_t *, const Vector &coeff, + real_t *y); + + /// parameters: use DIM, T_D1D, T_Q1D + MFEM_REGISTER_KERNELS(AssembleKernels, AssembleKernelType, (int, int, int)); + struct Kernels { Kernels(); }; + + template static void AddSpecialization() + { + AssembleKernels::Specialization::Add(); + } }; /// Class for domain integrator $ L(v) := (f, \nabla v) $ @@ -256,14 +272,13 @@ private: public: /// Constructs a domain integrator with a given VectorCoefficient - VectorDomainLFIntegrator(VectorCoefficient &QF) - : DeltaLFIntegrator(QF), Q(QF) { } + VectorDomainLFIntegrator(VectorCoefficient &QF, + const IntegrationRule *ir = nullptr); bool SupportsDevice() const override { return true; } /// Method defining assembly on device - void AssembleDevice(const FiniteElementSpace &fes, - const Array &markers, + void AssembleDevice(const FiniteElementSpace &fes, const Array &markers, Vector &b) override; /** Given a particular Finite Element and a transformation (Tr) @@ -277,6 +292,12 @@ public: Vector &elvect) override; using LinearFormIntegrator::AssembleRHSElementVect; + + template static void AddSpecialization() + { + // uses the same kernels for assembly + DomainLFIntegrator::AssembleKernels::Specialization::Add(); + } }; /** Class for domain integrator $ L(v) := (f, \nabla v) $, where @@ -544,7 +565,7 @@ public: Specifically, given the Dirichlet data $u_D$, the linear form assembles the following integrals on the boundary: $$ - \sigma \langle u_D, (Q \nabla v)) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle, + \sigma \langle u_D, (Q \nabla v) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle, $$ where Q is a scalar or matrix diffusion coefficient and v is the test function. The parameters $\sigma$ and $\kappa$ should be the same as the ones diff --git a/fem/lor/lor_batched.cpp b/fem/lor/lor_batched.cpp index d44dbc7d80..a54cfd756c 100644 --- a/fem/lor/lor_batched.cpp +++ b/fem/lor/lor_batched.cpp @@ -14,9 +14,11 @@ #include "../../general/forall.hpp" #include #include "../pbilinearform.hpp" +#include "../../fem/fe/face_map_utils.hpp" // Specializations #include "lor_h1.hpp" +#include "lor_dg.hpp" #include "lor_nd.hpp" #include "lor_rt.hpp" @@ -54,17 +56,18 @@ bool BatchedLORAssembly::FormIsSupported(BilinearForm &a) // Batched LOR requires all tensor elements if (!UsesTensorBasis(*a.FESpace())) { return false; } - if (dynamic_cast(fec)) + if (dynamic_cast(fec) || + dynamic_cast(fec)) { - if (HasIntegrators(a)) { return true; } + return HasIntegrators(a); } else if (dynamic_cast(fec)) { - if (HasIntegrators(a)) { return true; } + return HasIntegrators(a); } else if (dynamic_cast(fec)) { - if (HasIntegrators(a)) { return true; } + return HasIntegrators(a); } return false; } @@ -75,12 +78,14 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho, Mesh &mesh_ho = *fes_ho.GetMesh(); mesh_ho.EnsureNodes(); + const bool dg = fes_ho.IsDGSpace(); + // Get nodal points at the LOR vertices const int dim = mesh_ho.Dimension(); const int sdim = mesh_ho.SpaceDimension(); const int nel_ho = mesh_ho.GetNE(); const int order = fes_ho.GetMaxElementOrder(); - const int nd1d = order + 1; + const int nd1d = dg ? order + 2 : order + 1; const int ndof_per_el = static_cast(pow(nd1d, dim)); const GridFunction *nodal_gf = mesh_ho.GetNodes(); @@ -92,7 +97,8 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho, Vector nodal_evec(nodal_restriction->Height()); nodal_restriction->Mult(*nodal_gf, nodal_evec); - IntegrationRule ir = GetCollocatedIntRule(fes_ho); + const IntegrationRule ir = GetLobattoIntRule( + mesh_ho.GetTypicalElementGeometry(), nd1d); // Map from nodal E-vector to Q-vector at the LOR vertex points X_vert.SetSize(sdim*ndof_per_el*nel_ho); @@ -159,6 +165,7 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const const auto K = dof_glob2loc_offsets_.Read(); const auto map = Reshape(sparse_mapping.Read(), nnz_per_row, ndof_per_el); + auto I = A.WriteI(); mfem::forall(nvdof + 1, [=] MFEM_HOST_DEVICE (int ii) { I[ii] = 0; }); @@ -358,6 +365,177 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const }); } +void BatchedLORAssembly::SparseIJToCSR_DG(OperatorHandle &A) const +{ + const int ndof_per_el = fes_ho.GetFE(0)->GetDof(); + const int nel_ho = fes_ho.GetNE(); + const int nnz_per_row = sparse_ij.Size()/ndof_per_el/nel_ho; + const int dim = fes_ho.GetMesh()->Dimension(); + const int nrows = nel_ho*ndof_per_el; + const int p = fes_ho.GetMaxElementOrder(); + const int pp1 = p + 1; + const int nnz = nrows*nnz_per_row; + + const int face_nbr_vsize = [&]() + { +#ifdef MFEM_USE_MPI + if (auto *par_fes = dynamic_cast(&fes_ho)) + { + return par_fes->GetFaceNbrVSize(); + } +#endif + return 0; + }(); + + // If A contains an existing SparseMatrix, reuse it (and try to reuse its + // I, J, A arrays if they are big enough) + SparseMatrix *A_mat = A.Is(); + if (!A_mat) + { + A_mat = new SparseMatrix; + A.Reset(A_mat); + } + + // The second argument (nrows + face_nbr_vsize) accounts for additional + // columns contributed by DG face neighbors in parallel finite element + // spaces. In serial, face_nbr_vsize is set to 0. + A_mat->OverrideSize(nrows, nrows + face_nbr_vsize); + + EnsureCapacity(A_mat->GetMemoryI(), nrows + 1); + EnsureCapacity(A_mat->GetMemoryJ(), nnz); + EnsureCapacity(A_mat->GetMemoryData(), nnz); + + Array nbr_info(nel_ho*3*2*dim); + auto h_nbr_info = Reshape(nbr_info.HostWrite(), nel_ho, 2*dim, 3); + const int num_faces = fes_ho.GetMesh()->GetNumFaces(); + for (int f = 0; f < num_faces; f++) + { + Mesh::FaceInformation finfo = fes_ho.GetMesh()->GetFaceInformation(f); + int e0 = finfo.element[0].index; + int f0 = finfo.element[0].local_face_id; + if (finfo.IsBoundary()) + { + h_nbr_info(e0,f0,0) = -1; + h_nbr_info(e0,f0,1)= -1; + h_nbr_info(e0,f0,2)= -1; + } + else if (finfo.IsShared()) + { + // Face neighbors elements are indexed after the last local element + h_nbr_info(e0,f0,0) = nel_ho + finfo.element[1].index; + h_nbr_info(e0,f0,1)= finfo.element[1].orientation; + h_nbr_info(e0,f0,2)= finfo.element[1].local_face_id; + } + else if (finfo.IsInterior()) + { + int e1 = finfo.element[1].index; + int f1 = finfo.element[1].local_face_id; + h_nbr_info(e0,f0,0) = e1; + h_nbr_info(e0,f0,1)= finfo.element[1].orientation; + h_nbr_info(e0,f0,2)= f1; + h_nbr_info(e1,f1,0) = e0; + h_nbr_info(e1,f1,1) = finfo.element[1].orientation; + h_nbr_info(e1,f1,2) = f0; + } + }; + + auto h_I = A_mat->HostWriteI(); + h_I[0] = 0; + for (int i = 0; i < nrows; ++i) + { + const int iel_ho = i / ndof_per_el; + const int iloc = i % ndof_per_el; + static const int lex_map_2[4] = {3, 1, 0, 2}; + static const int lex_map_3[6] = {4, 2, 1, 3, 0, 5}; + const int local_i[3] = {iloc % pp1, (iloc/pp1)%pp1, iloc/pp1/pp1}; + int bdr_count = 0; + for (int n_idx = 0; n_idx < dim; ++n_idx) + { + for (int e_i = 0; e_i < 2; ++e_i) + { + const int j_lex = e_i + n_idx*2; + const int f = (dim == 3) ? lex_map_3[j_lex]:lex_map_2[j_lex]; + const bool boundary = (local_i[n_idx] == e_i * p); + if (boundary) + { + int neighbor_idx = h_nbr_info(iel_ho, f, 0); + if (neighbor_idx == -1) + { + ++bdr_count; + } + } + } + } + h_I[i+1] = h_I[i] + (nnz_per_row - bdr_count); + } + + const auto V = Reshape(sparse_ij.Read(), nnz_per_row, ndof_per_el, nel_ho); + auto J = A_mat->WriteJ(); + auto AV = A_mat->WriteData(); + auto I = A_mat->ReadI(); + + auto d_nbr_info = Reshape(nbr_info.Read(), nel_ho, 2*dim, 3); + mfem::forall(nrows, [=] MFEM_HOST_DEVICE (int i) + { + const int e = i / ndof_per_el; + const int iloc = i % ndof_per_el; + const int local_x = iloc % pp1; + const int local_y = (iloc/pp1)%pp1; + const int local_z = iloc/pp1/pp1; + const int local_i[3] = {local_x, local_y, local_z}; + int offset = I[i]; + static const int lex_map_2[4] = {3, 1, 0, 2}; + static const int lex_map_3[6] = {4,2,1,3,0,5}; + const int *lex_map = (dim == 2) ? lex_map_2 : lex_map_3; + AV[offset] = V(0, iloc, e); + J[offset] = i; + ++offset; + for (int n_idx = 0; n_idx < dim; ++n_idx) + { + // qi is the face lexicographic index, obtained by taking the + // lexicographic index of the coordinates ommiting n_idx. + int qi = 0; + int stride = 1; + for (int d = 0; d < dim; ++d) + { + if (d != n_idx) + { + qi += local_i[d]*stride; + stride *= pp1; + } + } + for (int e_i = 0; e_i < 2; ++e_i) + { + const int j_lex = e_i + n_idx*2; + const int f = lex_map[j_lex]; + const bool bdr = (local_i[n_idx] == e_i * p); + if (bdr) + { + const int nbr_e = d_nbr_info(e, f, 0); + const int nbr_ori = d_nbr_info(e, f, 1); + const int nbr_f = d_nbr_info(e, f, 2); + if (nbr_e != -1) + { + const int nbr_loc_idx = internal::FaceIdxToVolIdx( + dim, qi, pp1, f, nbr_f, 1, nbr_ori); + J[offset] = nbr_e*ndof_per_el + nbr_loc_idx; + AV[offset] = V(f+1, iloc, e); + ++offset; + } + } + else + { + int shift = (e_i == 0) ? -1 : 1; + for (int n = 0; n < n_idx; ++n) { shift *= pp1; } + J[offset] = i + shift; + AV[offset] = V(f+1, iloc, e); + ++offset; + } + } + } + }); +} + void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const { const int nvdof = fes_ho.GetVSize(); @@ -372,12 +550,11 @@ void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const } A_mat->OverrideSize(nvdof, nvdof); + EnsureCapacity(A_mat->GetMemoryI(), nvdof + 1); - A_mat->GetMemoryI().New(nvdof+1, Device::GetDeviceMemoryType()); - int nnz = FillI(*A_mat); - - A_mat->GetMemoryJ().New(nnz, Device::GetDeviceMemoryType()); - A_mat->GetMemoryData().New(nnz, Device::GetDeviceMemoryType()); + const int nnz = FillI(*A_mat); + EnsureCapacity(A_mat->GetMemoryJ(), nnz); + EnsureCapacity(A_mat->GetMemoryData(), nnz); FillJAndData(*A_mat); } @@ -431,6 +608,19 @@ void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A) // Assemble the matrix, depending on what the form is. // This fills in the arrays sparse_ij and sparse_mapping. const FiniteElementCollection *fec = fes_ho.FEColl(); + + // Handle DG case separately, because assembly of CSR matrix requires + // handling face terms. + if (dynamic_cast(fec)) + { + if (HasIntegrators(a)) + { + AssemblyKernel(a); + } + SparseIJToCSR_DG(A); + return; + } + if (dynamic_cast(fec)) { if (HasIntegrators(a)) @@ -453,10 +643,47 @@ void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A) } } - return SparseIJToCSR(A); + SparseIJToCSR(A); } #ifdef MFEM_USE_MPI +void BatchedLORAssembly::ParAssemble_DG(SparseMatrix &A_local, + OperatorHandle &A) +{ + auto &par_fes = static_cast(fes_ho); + + // handle the case when 'a' contains off-diagonal + const int lvsize = par_fes.GetVSize(); + const Array &face_nbr_glob_ldof = + par_fes.GetFaceNbrGlobalDofMapArray(); + const HYPRE_BigInt ldof_offset = par_fes.GetMyDofOffset(); + + const int nnz_local = A_local.NumNonZeroElems(); + Array glob_J(nnz_local); + + const HYPRE_BigInt *d_face_nbr_glob_ldof = face_nbr_glob_ldof.Read(); + const int *d_J = A_local.ReadJ(); + HYPRE_BigInt *d_glob_J = glob_J.Write(); + + mfem::forall(nnz_local, [=] MFEM_HOST_DEVICE (int i) + { + if (d_J[i] < lvsize) + { + d_glob_J[i] = d_J[i] + ldof_offset; + } + else + { + d_glob_J[i] = d_face_nbr_glob_ldof[d_J[i] - lvsize]; + } + }); + + A.Reset(new HypreParMatrix( + par_fes.GetComm(), lvsize, par_fes.GlobalVSize(), + par_fes.GlobalVSize(), A_local.HostReadWriteI(), + glob_J.HostReadWrite(), A_local.HostReadWriteData(), + par_fes.GetDofOffsets(), par_fes.GetDofOffsets())); +} + void BatchedLORAssembly::ParAssemble( BilinearForm &a, const Array &ess_dofs, OperatorHandle &A) { @@ -464,13 +691,18 @@ void BatchedLORAssembly::ParAssemble( OperatorHandle A_local; AssembleWithoutBC(a, A_local); - ParBilinearForm *pa = - dynamic_cast(&a); - - pa->ParallelRAP(*A_local.As(), A, true); - - A.As()->EliminateBC(ess_dofs, - Operator::DiagonalPolicy::DIAG_ONE); + if (dynamic_cast(fes_ho.FEColl())) + { + ParAssemble_DG(*A_local.As(), A); + } + else + { + ParBilinearForm *pa = + dynamic_cast(&a); + pa->ParallelRAP(*A_local.As(), A, true); + A.As()->EliminateBC(ess_dofs, + Operator::DiagonalPolicy::DIAG_ONE); + } } #endif @@ -504,12 +736,22 @@ BatchedLORAssembly::BatchedLORAssembly(FiniteElementSpace &fes_ho_) FormLORVertexCoordinates(fes_ho, X_vert); } -IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes) +IntegrationRule GetLobattoIntRule(Geometry::Type geom, int nd1d) { IntegrationRules irs(0, Quadrature1D::GaussLobatto); - const Geometry::Type geom = fes.GetMesh()->GetTypicalElementGeometry(); - const int nd1d = fes.GetMaxElementOrder() + 1; return irs.Get(geom, 2*nd1d - 3); } +IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes) +{ + const Geometry::Type geom = fes.GetMesh()->GetTypicalElementGeometry(); + return GetLobattoIntRule(geom, fes.GetMaxElementOrder() + 1); +} + +IntegrationRule GetCollocatedFaceIntRule(FiniteElementSpace &fes) +{ + const Geometry::Type geom = fes.GetMesh()->GetTypicalFaceGeometry(); + return GetLobattoIntRule(geom, fes.GetMaxElementOrder() + 1); +} + } // namespace mfem diff --git a/fem/lor/lor_batched.hpp b/fem/lor/lor_batched.hpp index d49d951bec..b99502d2e7 100644 --- a/fem/lor/lor_batched.hpp +++ b/fem/lor/lor_batched.hpp @@ -25,6 +25,7 @@ namespace mfem /// supported, currently: /// /// - H1 diffusion + mass +/// - DG diffusion + mass (in progress) /// - ND curl-curl + mass /// - RT div-div + mass /// @@ -73,6 +74,9 @@ public: /// Return the vertices of the LOR mesh in E-vector format const Vector &GetLORVertexCoordinates() { return X_vert; } + /// Specialized implementation of SparseIJToCSR for DG spaces. + void SparseIJToCSR_DG(OperatorHandle &A) const; + protected: /// After assembling the "sparse IJ" format, convert it to CSR. void SparseIJToCSR(OperatorHandle &A) const; @@ -105,6 +109,9 @@ public: void FillJAndData(SparseMatrix &A) const; #ifdef MFEM_USE_MPI + /// Assemble the parallel DG matrix (with shared faces). + void ParAssemble_DG(SparseMatrix &A_local, OperatorHandle &A); + /// Assemble the system in parallel and place the result in @a A. void ParAssemble(BilinearForm &a, const Array &ess_dofs, OperatorHandle &A); @@ -128,9 +135,8 @@ void EnsureCapacity(Memory &mem, int capacity) /// Return the first domain integrator in the form @a i of type @a T. template -static T *GetIntegrator(BilinearForm &a) +static T *GetIntegrator(Array *integs) { - Array *integs = a.GetDBFI(); if (integs != NULL) { for (auto *i : *integs) @@ -144,8 +150,32 @@ static T *GetIntegrator(BilinearForm &a) return nullptr; } +template +static T *GetIntegrator(BilinearForm &a) +{ + return GetIntegrator(a.GetDBFI()); +} + +template +static T *GetInteriorFaceIntegrator(BilinearForm &a) +{ + return GetIntegrator(a.GetFBFI()); +} + +/// @brief Return the Gauss-Lobatto rule for geometry @a geom with @a nd1d +/// points per dimension. +IntegrationRule GetLobattoIntRule(Geometry::Type geom, int nd1d); + +/// @brief Return the Gauss-Lobatto rule collocated with the element nodes. +/// +/// Assumes @a fes uses Gauss-Lobatto basis. IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes); +/// @brief Return the Gauss-Lobatto rule collocated with face nodes. +/// +/// Assumes @a fes uses Gauss-Lobatto basis. +IntegrationRule GetCollocatedFaceIntRule(FiniteElementSpace &fes); + template void ProjectLORCoefficient(BilinearForm &a, CoefficientVector &coeff_vector) { diff --git a/fem/lor/lor_dg.hpp b/fem/lor/lor_dg.hpp new file mode 100644 index 0000000000..9e1410f4bc --- /dev/null +++ b/fem/lor/lor_dg.hpp @@ -0,0 +1,79 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_LOR_DG +#define MFEM_LOR_DG + +#include "lor_batched.hpp" + +namespace mfem +{ + +// BatchedLORKernel specialization for DG spaces. Not user facing. See the +// classes BatchedLORAssembly and BatchedLORKernel . +class BatchedLOR_DG : BatchedLORKernel +{ + IntegrationRule ir_face; ///< Collocated Gauss-Lobatto face quadrature rule. + real_t kappa; ///< DG penalty parameter. +public: + template void Assemble2D(); + template void Assemble3D(); + BatchedLOR_DG(BilinearForm &a, + FiniteElementSpace &fes_ho_, + Vector &X_vert_, + Vector &sparse_ij_, + Array &sparse_mapping_) + : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_), + ir_face(GetLobattoIntRule(fes_ho_.GetMesh()->GetTypicalFaceGeometry(), + fes_ho_.GetMaxElementOrder() + 1)) + { + ProjectLORCoefficient(a, c1); + ProjectLORCoefficient(a, c2); + + auto *integ = GetInteriorFaceIntegrator(a); + if (integ) + { + kappa = integ->GetPenaltyParameter(); + } + else + { + kappa = 0.0; + } + } + + /// @brief Compute and return the face info array. + /// + /// The face info array has shape (6, nf), where @a nf is the number of + /// faces. For each face @a i, the column (:,i) has entries (e0, f0, o0, e1, + /// f1, o1), where @a e is adjacent element, @a f is the local face index, + /// and @a o is the orientation. For boundary and shared faces, (e1, f1, o1) + /// are all set to -1. + Array GetFaceInfo() const; + + /// @brief Compute and return the boundary penalty factor. + /// + /// The returned vector has shape (nq, nf), where @a nq is the number of + /// nodes per face, and @a nf is the number of faces. + /// + /// The boundary penalty factor is $J_f / h = J_f^2 / J_e$ (since $h = J_e / + /// J_f$), where $J_f$ is the face Jacobian determinant, and $J_e$ is the + /// element Jacobian determinant. + Vector GetBdrPenaltyFactor() const; + + /// Assemble the face penalty terms in the matrix @a sparse_ij. + void AssembleFaceTerms(); +}; + +} + +#include "lor_dg_impl.hpp" + +#endif diff --git a/fem/lor/lor_dg_impl.hpp b/fem/lor/lor_dg_impl.hpp new file mode 100644 index 0000000000..9544afecb5 --- /dev/null +++ b/fem/lor/lor_dg_impl.hpp @@ -0,0 +1,392 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#pragma once + +#include "lor_dg.hpp" +#include "../fe/face_map_utils.hpp" +#include "../../linalg/dtensor.hpp" +#include "../../general/forall.hpp" + +namespace mfem +{ + +Array BatchedLOR_DG::GetFaceInfo() const +{ + Mesh &mesh = *fes_ho.GetMesh(); + const int nf = mesh.GetNumFaces(); + Array face_info(nf * 6); // (e0, f0, o0, e1, f1, o1) + auto h_face_info = Reshape(face_info.HostWrite(), 6, nf); + for (int f = 0; f < nf; ++f) + { + auto finfo = mesh.GetFaceInformation(f); + h_face_info(0, f) = finfo.element[0].index; + h_face_info(1, f) = finfo.element[0].local_face_id; + h_face_info(2, f) = finfo.element[0].orientation; + if (finfo.IsLocal()) // Interior, non-shared face + { + h_face_info(3, f) = finfo.element[1].index; + h_face_info(4, f) = finfo.element[1].local_face_id; + h_face_info(5, f) = finfo.element[1].orientation; + } + else + { + h_face_info(3, f) = -1; + h_face_info(4, f) = -1; + h_face_info(5, f) = -1; + } + } + return face_info; +} + +Vector BatchedLOR_DG::GetBdrPenaltyFactor() const +{ + Mesh &mesh = *fes_ho.GetMesh(); + + const int nf = mesh.GetNumFaces(); + Array f_int(mesh.GetNFbyType(FaceType::Interior)); + Array f_bdr(mesh.GetNFbyType(FaceType::Boundary)); + { + int i_int = 0; + int i_bdr = 0; + for (int i = 0; i < nf; ++i) + { + const auto f = mesh.GetFaceInformation(i); + if (f.IsBoundary()) + { + f_bdr[i_bdr] = i; + ++i_bdr; + } + else if (f.IsInterior()) + { + f_int[i_int] = i; + ++i_int; + } + } + } + + const auto geom = fes_ho.GetMesh()->GetGeometricFactors( + ir, GeometricFactors::DETERMINANTS); + + const int nq = ir_face.Size(); + Vector face_Jh(nq * nf); + for (const FaceType ft : {FaceType::Interior, FaceType::Boundary}) + { + const int nft = mesh.GetNFbyType(ft); + auto *geom_face = mesh.GetFaceGeometricFactors( + ir_face, FaceGeometricFactors::DETERMINANTS, ft); + + const L2FaceValues fv = (ft == FaceType::Interior) + ? L2FaceValues::DoubleValued + : L2FaceValues::SingleValued; + const int m = (fv == L2FaceValues::DoubleValued) ? 2 : 1; + + auto *r = fes_ho.GetFaceRestriction(ElementDofOrdering::LEXICOGRAPHIC, ft, fv); + Vector detJ_r(nq * m * nft); + r->Mult(geom->detJ, detJ_r); + + const auto *d_i = (ft == FaceType::Interior) ? f_int.Read() : f_bdr.Read(); + const auto d_detJ_face = Reshape(geom_face->detJ.Read(), nq, nft); + const auto d_detJ_r = Reshape(detJ_r.Read(), nq, m, nft); + auto d_face_Jh = Reshape(face_Jh.Write(), nq, nf); + + mfem::forall(nft * nq, [=] MFEM_HOST_DEVICE (int ii) + { + const int i = ii % nq; + const int f = ii / nq; + const real_t J_el = 0.5*(d_detJ_r(i, 0, f) + d_detJ_r(i, m==2?1:0, f)); + const real_t J_f = d_detJ_face(i, f); + d_face_Jh(i, d_i[f]) = J_f * J_f / J_el; + }); + } + return face_Jh; +} + +void BatchedLOR_DG::AssembleFaceTerms() +{ + Mesh &mesh = *fes_ho.GetMesh(); + + const int nnz_per_row = 1 + mesh.Dimension()*2; + const int pp1 = fes_ho.GetMaxElementOrder() + 1; + const int nel_ho = mesh.GetNE(); + const int nf = mesh.GetNumFaces(); + const int nd_face = ir_face.Size(); + const int nd = ir.Size(); + const int dim = mesh.Dimension(); + + Array face_info = GetFaceInfo(); + const auto d_face_info = Reshape(face_info.Read(), 6, nf); + + Vector face_Jh = GetBdrPenaltyFactor(); + const auto d_face_Jh = Reshape(face_Jh.Read(), nd_face, nf); + + const auto *w_face = ir_face.GetWeights().Read(); + + // Penalty parameter (avoid capturing *this in lambda) + const real_t d_kappa = kappa; + + // Get diffusion coefficient + const bool const_dq = c2.Size() == 1; + const auto DQ = const_dq?Reshape(c2.Read(),1,1):Reshape(c2.Read(),nd,nel_ho); + + // Sparse matrix entries + auto V = Reshape(sparse_ij.ReadWrite(), nnz_per_row, nd, nel_ho); + + mfem::forall(nf, [=] MFEM_HOST_DEVICE (int f) + { + const int f_0 = d_face_info(1, f); + const int f_1 = d_face_info(4, f); + const int nsides = (f_1 >= 0) ? 2 : 1; + for (int el_i = 0; el_i < nsides; ++el_i) + { + const int e = d_face_info(3*el_i, f); + const int o = d_face_info(3*el_i + 2, f); + const int v_idx = 1 + ((el_i == 0) ? f_0 : f_1); + for (int i = 0; i < nd_face; ++i) + { + const int ii = internal::FaceIdxToVolIdx(dim, i, pp1, f_0, f_1, el_i, o); + const real_t Jh = d_face_Jh(i, f); + const real_t dq = const_dq ? DQ(0,0) : DQ(ii, e); + V(v_idx, ii, e) = -dq*d_kappa*Jh*w_face[i]; + } + } + }); +} + +template +void BatchedLOR_DG::Assemble2D() +{ + MFEM_VERIFY(SDIM == 2, "Surface meshes not currently supported for LOR-DG.") + + static constexpr int pp1 = ORDER + 1; + static constexpr int ndof_per_el = pp1*pp1; + static constexpr int nnz_per_row = 5; + const int nel_ho = fes_ho.GetNE(); + + // Get element geometric factors; calling before AssembleFaceTerms, since + // in AssembleFaceTerms, element Jacobian determinants are used, potentially + // saving recomputation. + const auto factors = GeometricFactors::DETERMINANTS | + GeometricFactors::JACOBIANS; + const auto *geom = fes_ho.GetMesh()->GetGeometricFactors(ir, factors); + + // Sparse matrix entries + sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho); + sparse_ij.UseDevice(true); + sparse_ij = 0.0; + auto V = Reshape(sparse_ij.ReadWrite(), nnz_per_row, pp1, pp1, nel_ho); + + AssembleFaceTerms(); + + // Populate Gauss-Lobatto quadrature rule of size (p+1) + IntegrationRule ir_pp1; + QuadratureFunctions1D::GaussLobatto(pp1, &ir_pp1); + Vector glx_pp1(pp1), glw_pp1(pp1); + for (int i = 0; i < pp1; ++i) + { + glx_pp1[i] = ir_pp1[i].x; + glw_pp1[i] = ir_pp1[i].weight; + } + const auto *x_pp1 = glx_pp1.Read(); + const auto *w_1d = glw_pp1.Read(); + + // Get coefficients for mass and diffusion + const bool const_mq = c1.Size() == 1; + const auto MQ = const_mq + ? Reshape(c1.Read(), 1, 1, 1) + : Reshape(c1.Read(), pp1, pp1, nel_ho); + const bool const_dq = c2.Size() == 1; + const auto DQ = const_dq + ? Reshape(c2.Read(), 1, 1, 1) + : Reshape(c2.Read(), pp1, pp1, nel_ho); + + const auto detJ = Reshape(geom->detJ.Read(), pp1, pp1, nel_ho); + const auto J = Reshape(geom->J.Read(), pp1, pp1, 2, 2, nel_ho); + const auto W = Reshape(ir.GetWeights().Read(), pp1, pp1); + + mfem::forall(nel_ho, [=] MFEM_HOST_DEVICE (int iel_ho) + { + for (int iy = 0; iy < pp1; ++iy) + { + for (int ix = 0; ix < pp1; ++ix) + { + const real_t mq = const_mq ? MQ(0,0,0) : MQ(ix, iy, iel_ho); + const real_t dq = const_dq ? DQ(0,0,0) : DQ(ix, iy, iel_ho); + + for (int n_idx = 0; n_idx < 2; ++n_idx) + { + for (int e_i = 0; e_i < 2; ++e_i) + { + const int i_0 = (n_idx == 0) ? ix + e_i : ix; + const int j_0 = (n_idx == 1) ? iy + e_i : iy; + + const bool bdr = (n_idx == 0 && (i_0 == 0 || i_0 == pp1)) || + (n_idx == 1 && (j_0 == 0 || j_0 == pp1)); + + if (bdr) { continue; } + + static constexpr int lex_map[] = {4, 2, 1, 3}; + const int v_idx_lex = e_i + n_idx*2; + const int v_idx = lex_map[v_idx_lex]; + + const int w_idx = (n_idx == 0) ? iy : ix; + const int x_idx = (n_idx == 0) ? i_0 : j_0; + + const real_t J1 = J(ix, iy, n_idx, !n_idx, iel_ho); + const real_t J2 = J(ix, iy, !n_idx, !n_idx, iel_ho); + const real_t Jh = (J1*J1 + J2*J2) / detJ(ix, iy, iel_ho); + + V(v_idx, ix, iy, iel_ho) = + -dq * Jh * w_1d[w_idx] / (x_pp1[x_idx] - x_pp1[x_idx -1]); + } + } + V(0, ix, iy, iel_ho) = mq * detJ(ix, iy, iel_ho) * W(ix, iy); + for (int i = 1; i < nnz_per_row; ++i) + { + V(0, ix, iy, iel_ho) -= V(i, ix, iy, iel_ho); + } + } + } + }); +} + +template +void BatchedLOR_DG::Assemble3D() +{ + static constexpr int pp1 = ORDER + 1; + static constexpr int ndof_per_el = pp1*pp1*pp1; + static constexpr int nnz_per_row = 7; + const int nel_ho = fes_ho.GetNE(); + + // Get element geometric factors; calling before AssembleFaceTerms, since + // in AssembleFaceTerms, element Jacobian determinants are used, potentially + // saving recomputation. + const auto factors = GeometricFactors::DETERMINANTS | + GeometricFactors::JACOBIANS; + const auto geom = fes_ho.GetMesh()->GetGeometricFactors(ir, factors); + + sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho); + sparse_ij.UseDevice(true); + sparse_ij = 0.0; + auto V = Reshape(sparse_ij.Write(), nnz_per_row, pp1, pp1, pp1, nel_ho); + + AssembleFaceTerms(); + + // Populate Gauss-Lobatto quadrature rule of size (p+1) + IntegrationRule ir_pp1; + QuadratureFunctions1D::GaussLobatto(pp1, &ir_pp1); + Vector glx_pp1(pp1), glw_pp1(pp1); + for (int i = 0; i < pp1; ++i) + { + glx_pp1[i] = ir_pp1[i].x; + glw_pp1[i] = ir_pp1[i].weight; + } + const auto *x_pp1 = glx_pp1.Read(); + const auto *w_1d = glw_pp1.Read(); + + const bool const_mq = c1.Size() == 1; + const auto MQ = const_mq + ? Reshape(c1.Read(), 1, 1, 1, 1) + : Reshape(c1.Read(), pp1, pp1, pp1, nel_ho); + const bool const_dq = c2.Size() == 1; + const auto DQ = const_dq + ? Reshape(c2.Read(), 1, 1, 1, 1) + : Reshape(c2.Read(), pp1, pp1, pp1, nel_ho); + const auto W = Reshape(ir.GetWeights().Read(), pp1, pp1, pp1); + + const auto detJ = Reshape(geom->detJ.Read(), pp1, pp1, pp1, nel_ho); + const auto J = Reshape(geom->J.Read(), pp1, pp1, pp1, 3, 3, nel_ho); + + mfem::forall(nel_ho, [=] MFEM_HOST_DEVICE (int iel_ho) + { + for (int iz = 0; iz < pp1; ++iz) + { + for (int iy = 0; iy < pp1; ++iy) + { + for (int ix = 0; ix < pp1; ++ix) + { + const real_t mq = const_mq ? MQ(0,0,0,0) : MQ(ix, iy, iz, iel_ho); + const real_t dq = const_dq ? DQ(0,0,0,0) : DQ(ix, iy, iz, iel_ho); + + const real_t DETJ = detJ(ix, iy, iz, iel_ho); + + for (int n_idx = 0; n_idx < 3; ++n_idx) + { + for (int e_i = 0; e_i < 2; ++e_i) + { + static constexpr int lex_map[] = {5,3,2,4,1,6}; + const int v_idx_lex = e_i + n_idx*2; + const int v_idx = lex_map[v_idx_lex]; + + const int i_0 = (n_idx == 0) ? ix + e_i : ix; + const int j_0 = (n_idx == 1) ? iy + e_i : iy; + const int k_0 = (n_idx == 2) ? iz + e_i : iz; + + const bool bdr = + (n_idx == 0 && (i_0 == 0 || i_0 == pp1)) || + (n_idx == 1 && (j_0 == 0 || j_0 == pp1)) || + (n_idx == 2 && (k_0 == 0 || k_0 == pp1)); + + if (bdr) { continue; } + + int x_idx = (n_idx == 0) ? i_0 : (n_idx == 1) ? j_0 : k_0; + int w_idx_1 = (n_idx == 0) ? iy : (n_idx == 1) ? iz : ix; + int w_idx_2 = (n_idx == 0) ? iz : (n_idx == 1) ? ix : iy; + + const real_t J00 = J(ix, iy, iz, 0, 0, iel_ho); + const real_t J01 = J(ix, iy, iz, 0, 1, iel_ho); + const real_t J02 = J(ix, iy, iz, 0, 2, iel_ho); + const real_t J10 = J(ix, iy, iz, 1, 0, iel_ho); + const real_t J11 = J(ix, iy, iz, 1, 1, iel_ho); + const real_t J12 = J(ix, iy, iz, 1, 2, iel_ho); + const real_t J20 = J(ix, iy, iz, 2, 0, iel_ho); + const real_t J21 = J(ix, iy, iz, 2, 1, iel_ho); + const real_t J22 = J(ix, iy, iz, 2, 2, iel_ho); + + real_t JinvJinvT_diag = 0.0; + if (n_idx == 0) + { + JinvJinvT_diag = J02*J02*(J11*J11 + J21*J21) + (J12*J21 - J11*J22)* + (J12*J21 - J11*J22) - 2*J01*J02*(J11*J12 + J21*J22) + J01*J01* + (J12*J12 + J22*J22); + } + else if (n_idx == 1) + { + JinvJinvT_diag = J02*J02*(J10*J10 + J20*J20) + (J12*J20 - J10*J22)* + (J12*J20 - J10*J22) - 2*J00*J02*(J10*J12 + J20*J22) + J00*J00* + (J12*J12 + J22*J22); + } + else if (n_idx == 2) + { + JinvJinvT_diag = J01*J01*(J10*J10 + J20*J20) + (J11*J20 - J10*J21)* + (J11*J20 - J10*J21) - 2*J00*J01*(J10*J11 + J20*J21) + J00*J00* + (J11*J11 + J21*J21); + } + + const real_t Jh = JinvJinvT_diag / DETJ; + + V(v_idx, ix, iy, iz, iel_ho) = -dq * Jh * w_1d[w_idx_1] * w_1d[w_idx_2] / + (x_pp1[x_idx] - x_pp1[x_idx -1]); + } + } + V(0, ix, iy, iz, iel_ho) = mq * DETJ * W(ix, iy, iz); + for (int i = 1; i < 7; ++i) + { + V(0, ix, iy, iz, iel_ho) -= V(i, ix, iy, iz, iel_ho); + } + } + } + } + + }); +} + +} // namespace mfem diff --git a/fem/nonlinearform.cpp b/fem/nonlinearform.cpp index 900e93cd11..3983381c9e 100644 --- a/fem/nonlinearform.cpp +++ b/fem/nonlinearform.cpp @@ -436,7 +436,7 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const // In parallel, the result is in 'py' which is an alias for 'aux2'. } -Operator &NonlinearForm::GetGradient(const Vector &x) const +Operator &NonlinearForm::GetGradient(const Vector &x, bool finalize) const { if (ext) { @@ -644,6 +644,8 @@ Operator &NonlinearForm::GetGradient(const Vector &x) const } } + if (!finalize) { return *Grad; } + if (!Grad->Finalized()) { Grad->Finalize(skip_zeros); @@ -788,12 +790,10 @@ BlockNonlinearForm::BlockNonlinearForm(Array &f) : } void BlockNonlinearForm::SetEssentialBC( - const Array *> &bdr_attr_is_ess, Array &rhs) + const Array*> &bdr_attr_is_ess, Array &rhs) { for (int s = 0; s < fes.Size(); ++s) { - ess_tdofs[s]->SetSize(ess_tdofs.Size()); - fes[s]->GetEssentialTrueDofs(*bdr_attr_is_ess[s], *ess_tdofs[s]); if (rhs[s]) @@ -803,6 +803,19 @@ void BlockNonlinearForm::SetEssentialBC( } } +void BlockNonlinearForm::SetEssentialTrueDofs( + const Array*> &ess_tdof_list, Array &rhs) +{ + for (int s = 0; s < fes.Size(); ++s) + { + *ess_tdofs[s] = *ess_tdof_list[s]; + if (rhs[s]) + { + rhs[s]->SetSubVector(*ess_tdofs[s], 0.0); + } + } +} + real_t BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const { Array *> vdofs(fes.Size()); @@ -1192,7 +1205,14 @@ const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const aux1.Update(block_offsets); for (int s = 0; s < fes.Size(); s++) { - P[s]->Mult(bx.GetBlock(s), aux1.GetBlock(s)); + if (P[s]) + { + P[s]->Mult(bx.GetBlock(s), aux1.GetBlock(s)); + } + else + { + aux1.GetBlock(s) = bx.GetBlock(s); + } } return aux1; } @@ -1221,11 +1241,16 @@ void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const { cP[s]->MultTranspose(pby.GetBlock(s), by.GetBlock(s)); } + else if (needs_prolongation) + { + by.GetBlock(s) = pby.GetBlock(s); + } by.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0); } } -void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const +void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx, + bool finalize) const { const int skip_zeros = 0; Array *> vdofs(fes.Size()); @@ -1479,7 +1504,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const } } - if (!Grads(0,0)->Finalized()) + if (finalize && !Grads(0,0)->Finalized()) { for (int i=0; i &bdr_marker) { bfnfi.Append(nlfi); bfnfi_marker.Append(&bdr_marker); } - virtual void SetEssentialBC(const Array *>&bdr_attr_is_ess, - Array &rhs); + /** @brief Set essential boundary conditions to each finite element space + using boundary attribute markers. + + This method calls `FiniteElementSpace::GetEssentialTrueDofs()` for each + space and stores ess_tdof_lists internally. + + If `rhs` vectors are non-null, the entries corresponding to these + essential DoFs are set to zero. This ensures compatibility with the + output of the `Mult()` method, which also zeroes out these entries. + + @param[in] bdr_attr_is_ess A list of boundary attribute markers for each + space. + @param[in,out] rhs An array of optional right-hand side vectors. + If a vector at `rhs[i]` is non-null, its essential DoFs will be set + to zero. */ + virtual void SetEssentialBC(const Array*> &bdr_attr_is_ess, + Array &rhs); + + /** @brief Set essential boundary conditions to each finite element space + using essential true dof lists. + + This method stores a copy of the provided essential true dof lists. + + If `rhs` vectors are non-null, the entries corresponding to these + essential DoFs are set to zero. This ensures compatibility with the + output of the `Mult()` method, which also zeroes out these entries. + + @param[in] ess_tdof_list A list of essential true dofs for each space. + @param[in,out] rhs An array of optional right-hand side vectors. + If a vector at `rhs[i]` is non-null, its essential DoFs will be set + to zero. */ + virtual void SetEssentialTrueDofs(const Array*> &ess_tdof_list, + Array &rhs); virtual real_t GetEnergy(const Vector &x) const; diff --git a/fem/pbilinearform.cpp b/fem/pbilinearform.cpp index 211b15b0cb..e948942903 100644 --- a/fem/pbilinearform.cpp +++ b/fem/pbilinearform.cpp @@ -151,6 +151,15 @@ void ParBilinearForm::ParallelRAP(SparseMatrix &loc_A, OperatorHandle &A, } } +HypreParMatrix *ParBilinearForm::ParallelAssembleInternalMatrix() +{ + if (p_mat.Ptr() == NULL) + { + ParallelAssemble(p_mat, mat); + } + return p_mat.As(); +} + void ParBilinearForm::ParallelAssemble(OperatorHandle &A, SparseMatrix *A_local) { A.Clear(); @@ -333,6 +342,15 @@ void ParBilinearForm A.EliminateRowsCols(dof_list, X, B); } +void ParBilinearForm::ParallelEliminateEssentialBC( + const Array &bdr_attr_is_ess, const HypreParVector &X, HypreParVector &B) +{ + Array dof_list; + pfes->GetEssentialTrueDofs(bdr_attr_is_ess, dof_list); + + p_mat.As()->EliminateRowsCols(dof_list, X, B); +} + HypreParMatrix *ParBilinearForm:: ParallelEliminateEssentialBC(const Array &bdr_attr_is_ess, HypreParMatrix &A) const @@ -344,6 +362,26 @@ ParallelEliminateEssentialBC(const Array &bdr_attr_is_ess, return A.EliminateRowsCols(dof_list); } +void ParBilinearForm::ParallelEliminateEssentialBC(const Array + &bdr_attr_is_ess) +{ + Array tdofs_list; + pfes->GetEssentialTrueDofs(bdr_attr_is_ess, tdofs_list); + + ParallelEliminateTDofs(tdofs_list); +} + +void ParBilinearForm::ParallelEliminateTDofs(const Array &tdofs_list) +{ + p_mat_e.EliminateRowsCols(p_mat, tdofs_list); +} + +void ParBilinearForm::ParallelEliminateTDofsInRHS( + const Array &tdofs_list, const Vector &x, Vector &b) +{ + p_mat.EliminateBC(p_mat_e, tdofs_list, x, b); +} + void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const real_t a) const { @@ -485,7 +523,7 @@ void ParBilinearForm::FormLinearSystem( HypreParVector true_X(pfes), true_B(pfes); P.MultTranspose(b, true_B); R.Mult(x, true_X); - p_mat.EliminateBC(p_mat_e, ess_tdof_list, true_X, true_B); + ParallelEliminateTDofsInRHS(ess_tdof_list, true_X, true_B); R.MultTranspose(true_B, b); hybridization->ReduceRHS(true_B, B); X.SetSize(B.Size()); @@ -498,17 +536,11 @@ void ParBilinearForm::FormLinearSystem( B.SetSize(X.Size()); P.MultTranspose(b, B); R.Mult(x, X); - p_mat.EliminateBC(p_mat_e, ess_tdof_list, X, B); + ParallelEliminateTDofsInRHS(ess_tdof_list, X, B); if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); } } } -void ParBilinearForm::EliminateVDofsInRHS( - const Array &vdofs, const Vector &x, Vector &b) -{ - p_mat.EliminateBC(p_mat_e, vdofs, x, b); -} - void ParBilinearForm::FormSystemMatrix(const Array &ess_tdof_list, OperatorHandle &A) { @@ -553,7 +585,7 @@ void ParBilinearForm::FormSystemMatrix(const Array &ess_tdof_list, mat = NULL; delete mat_e; mat_e = NULL; - p_mat_e.EliminateRowsCols(p_mat, ess_tdof_list); + ParallelEliminateTDofs(ess_tdof_list); } if (hybridization) { @@ -615,36 +647,180 @@ void ParBilinearForm::Update(FiniteElementSpace *nfes) p_mat_e.Clear(); } - -HypreParMatrix *ParMixedBilinearForm::ParallelAssemble() +void ParMixedBilinearForm::pAllocMat() { - // construct the block-diagonal matrix A - HypreParMatrix *A = - new HypreParMatrix(trial_pfes->GetComm(), - test_pfes->GlobalVSize(), - trial_pfes->GlobalVSize(), - test_pfes->GetDofOffsets(), - trial_pfes->GetDofOffsets(), - mat); + const int trial_nbr_size = trial_pfes->GetFaceNbrVSize(); + const int test_nbr_size = test_pfes->GetFaceNbrVSize(); - HypreParMatrix *rap = RAP(test_pfes->Dof_TrueDof_Matrix(), A, - trial_pfes->Dof_TrueDof_Matrix()); - - delete A; - - return rap; + if (keep_nbr_block) + { + mat = new SparseMatrix(height + test_nbr_size, width + trial_nbr_size); + } + else + { + mat = new SparseMatrix(height, width + trial_nbr_size); + } } -void ParMixedBilinearForm::ParallelAssemble(OperatorHandle &A) +void ParMixedBilinearForm::AssembleSharedFaces(int skip_zeros) { - // construct the rectangular block-diagonal matrix dA - OperatorHandle dA(A.Type()); - dA.MakeRectangularBlockDiag(trial_pfes->GetComm(), - test_pfes->GlobalVSize(), - trial_pfes->GlobalVSize(), - test_pfes->GetDofOffsets(), - trial_pfes->GetDofOffsets(), - mat); + ParMesh *pmesh = trial_pfes->GetParMesh(); + FaceElementTransformations *T; + Array tr_vdofs1, tr_vdofs2, tr_vdofs_all; + Array te_vdofs1, te_vdofs2, te_vdofs_all; + DenseMatrix elemmat; + + int nfaces = pmesh->GetNSharedFaces(); + for (int i = 0; i < nfaces; i++) + { + T = pmesh->GetSharedFaceTransformations(i); + int Elem2NbrNo = T->Elem2No - pmesh->GetNE(); + trial_pfes->GetElementVDofs(T->Elem1No, tr_vdofs1); + test_pfes->GetElementVDofs(T->Elem1No, te_vdofs1); + trial_pfes->GetFaceNbrElementVDofs(Elem2NbrNo, tr_vdofs2); + test_pfes->GetFaceNbrElementVDofs(Elem2NbrNo, te_vdofs2); + + tr_vdofs1.Copy(tr_vdofs_all); + for (int j = 0; j < tr_vdofs2.Size(); j++) + { + if (tr_vdofs2[j] >= 0) + { + tr_vdofs2[j] += width; + } + else + { + tr_vdofs2[j] -= width; + } + } + tr_vdofs_all.Append(tr_vdofs2); + + if (keep_nbr_block) + { + te_vdofs1.Copy(te_vdofs_all); + for (int j = 0; j < te_vdofs2.Size(); j++) + { + if (te_vdofs2[j] >= 0) + { + te_vdofs2[j] += height; + } + else + { + te_vdofs2[j] -= height; + } + } + te_vdofs_all.Append(te_vdofs2); + } + + for (int k = 0; k < interior_face_integs.Size(); k++) + { + interior_face_integs[k]-> + AssembleFaceMatrix(*trial_pfes->GetFE(T->Elem1No), + *test_pfes->GetFE(T->Elem1No), + *trial_pfes->GetFaceNbrFE(Elem2NbrNo), + *test_pfes->GetFaceNbrFE(Elem2NbrNo), + *T, elemmat); + if (keep_nbr_block) + { + mat->AddSubMatrix(te_vdofs_all, tr_vdofs_all, elemmat, skip_zeros); + } + else + { + mat->AddSubMatrix(te_vdofs1, tr_vdofs_all, elemmat, skip_zeros); + } + } + } +} + +void ParMixedBilinearForm::Assemble(int skip_zeros) +{ + if (interior_face_integs.Size()) + { + trial_pfes->ExchangeFaceNbrData(); + test_pfes->ExchangeFaceNbrData(); + if (!ext && mat == NULL) + { + pAllocMat(); + } + } + + MixedBilinearForm::Assemble(skip_zeros); + + if (!ext && interior_face_integs.Size() > 0) + { + AssembleSharedFaces(skip_zeros); + } +} + +HypreParMatrix *ParMixedBilinearForm::ParallelAssembleInternalMatrix() +{ + if (p_mat.Ptr() == NULL) + { + ParallelAssemble(p_mat, mat); + } + return p_mat.As(); +} + +HypreParMatrix *ParMixedBilinearForm::ParallelAssemble(SparseMatrix *m) +{ + OperatorHandle Mh(Operator::Hypre_ParCSR); + ParallelAssemble(Mh, m); + Mh.SetOperatorOwner(false); + return Mh.As(); +} + +void ParMixedBilinearForm::ParallelAssemble(OperatorHandle &A, + SparseMatrix *A_local) +{ + A.Clear(); + + if (A_local == NULL) { return; } + MFEM_VERIFY(A_local->Finalized(), "the local matrix must be finalized"); + + OperatorHandle dA(A.Type()), hdA; + + if (interior_face_integs.Size() == 0) + { + // construct the rectangular block-diagonal matrix dA + dA.MakeRectangularBlockDiag(trial_pfes->GetComm(), + test_pfes->GlobalVSize(), + trial_pfes->GlobalVSize(), + test_pfes->GetDofOffsets(), + trial_pfes->GetDofOffsets(), + A_local); + } + else + { + // handle the case when 'a' contains off-diagonal + const int lvrows = test_pfes->GetVSize(); + const int lvcols = trial_pfes->GetVSize(); + const HYPRE_BigInt *face_nbr_glob_lcol = trial_pfes->GetFaceNbrGlobalDofMap(); + const HYPRE_BigInt lcol_offset = trial_pfes->GetMyDofOffset(); + + Array glob_J(A_local->NumNonZeroElems()); + const int *J = A_local->GetJ(); + for (int i = 0; i < glob_J.Size(); i++) + { + if (J[i] < lvcols) + { + glob_J[i] = J[i] + lcol_offset; + } + else + { + glob_J[i] = face_nbr_glob_lcol[J[i] - lvcols]; + } + } + + // TODO - construct dA directly in the A format + hdA.Reset( + new HypreParMatrix(trial_pfes->GetComm(), lvrows, test_pfes->GlobalVSize(), + trial_pfes->GlobalVSize(), A_local->GetI(), glob_J, + A_local->GetData(), test_pfes->GetDofOffsets(), + trial_pfes->GetDofOffsets())); + // - hdA owns the new HypreParMatrix + // - the above constructor copies all input arrays + glob_J.DeleteAll(); + dA.ConvertFrom(hdA); + } OperatorHandle P_test(A.Type()), P_trial(A.Type()); @@ -670,6 +846,44 @@ void ParMixedBilinearForm::TrueAddMult(const Vector &x, Vector &y, test_pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Yaux, 1.0, y); } +void ParMixedBilinearForm::ParallelEliminateTrialEssentialBC( + const Array &bdr_attr_is_ess) +{ + Array trial_tdof_list; + trial_pfes->GetEssentialTrueDofs(bdr_attr_is_ess, trial_tdof_list); + + ParallelEliminateTrialTDofs(trial_tdof_list); +} + +void ParMixedBilinearForm::ParallelEliminateTrialTDofs( + const Array &trial_tdof_list) +{ + HypreParMatrix *temp = p_mat.As()->EliminateCols( + trial_tdof_list); + p_mat_e.Reset(temp, true); +} + +void ParMixedBilinearForm::ParallelEliminateTrialTDofsInRHS( + const Array &trial_tdof_list, const Vector &x, Vector &b) +{ + p_mat_e.As()->Mult(-1.0, x, 1.0, b); +} + +void ParMixedBilinearForm::ParallelEliminateTestEssentialBC( + const Array &bdr_attr_is_ess) +{ + Array test_tdof_list; + test_pfes->GetEssentialTrueDofs(bdr_attr_is_ess, test_tdof_list); + + ParallelEliminateTestTDofs(test_tdof_list); +} + +void ParMixedBilinearForm::ParallelEliminateTestTDofs( + const Array &test_tdof_list) +{ + p_mat.As()->EliminateRows(test_tdof_list); +} + void ParMixedBilinearForm::FormRectangularSystemMatrix( const Array &trial_tdof_list, @@ -690,10 +904,8 @@ void ParMixedBilinearForm::FormRectangularSystemMatrix( mat = NULL; delete mat_e; mat_e = NULL; - HypreParMatrix *temp = - p_mat.As()->EliminateCols(trial_tdof_list); - p_mat.As()->EliminateRows(test_tdof_list); - p_mat_e.Reset(temp, true); + ParallelEliminateTrialTDofs(trial_tdof_list); + ParallelEliminateTestTDofs(test_tdof_list); } A = p_mat; @@ -723,7 +935,7 @@ void ParMixedBilinearForm::FormRectangularLinearSystem( test_P->MultTranspose(b, B); trial_R->Mult(x, X); - p_mat_e.As()->Mult(-1.0, X, 1.0, B); + ParallelEliminateTrialTDofsInRHS(trial_tdof_list, X, B); B.SetSubVector(test_tdof_list, 0.0); } diff --git a/fem/pbilinearform.hpp b/fem/pbilinearform.hpp index a949d44d49..4e35be3f23 100644 --- a/fem/pbilinearform.hpp +++ b/fem/pbilinearform.hpp @@ -73,7 +73,7 @@ public: /** When set to true and the ParBilinearForm has interior face integrators, the local SparseMatrix will include the rows (in addition to the columns) corresponding to face-neighbor dofs. The default behavior is to disregard - those rows. Must be called before the first Assemble call. */ + those rows. Must be called before the first Assemble() call. */ void KeepNbrBlock(bool knb = true) { keep_nbr_block = knb; } /** @brief Set the operator type id for the parallel matrix/operator when @@ -101,6 +101,14 @@ public: diagonal for this case. */ void AssembleDiagonal(Vector &diag) const override; + /// Returns the matrix assembled on the true dofs, i.e. P^t A P. + /** The returned matrix is the internal one, owned by the form. It is not + reassembled if it has been already constructed. If FormSystemMatrix() + has been called before, it is the system matrix with eliminated + essential DOFs, otherwise the parallel matrix is assembled here without + the elimination process. */ + HypreParMatrix *ParallelAssembleInternalMatrix(); + /// Returns the matrix assembled on the true dofs, i.e. P^t A P. /** The returned matrix has to be deleted by the caller. */ HypreParMatrix *ParallelAssemble() { return ParallelAssemble(mat); } @@ -146,6 +154,13 @@ public: const HypreParVector &X, HypreParVector &B) const; + /// Eliminate essential boundary DOFs from the parallel system matrix. + /** The array @a bdr_attr_is_ess marks boundary attributes that constitute + the essential part of the boundary. */ + void ParallelEliminateEssentialBC(const Array &bdr_attr_is_ess, + const HypreParVector &X, + HypreParVector &B); + /// Eliminate essential boundary DOFs from a parallel assembled matrix @a A. /** The array @a bdr_attr_is_ess marks boundary attributes that constitute the essential part of the boundary. The eliminated part is stored in a @@ -157,6 +172,12 @@ public: HypreParMatrix *ParallelEliminateEssentialBC(const Array &bdr_attr_is_ess, HypreParMatrix &A) const; + /// Eliminate essential boundary DOFs from the parallel system matrix. + /** The array @a bdr_attr_is_ess marks boundary attributes that constitute + the essential part of the boundary. This method relies on + ParallelEliminateTDofs(const Array &), see it for details. */ + void ParallelEliminateEssentialBC(const Array &bdr_attr_is_ess); + /// Eliminate essential true DOFs from a parallel assembled matrix @a A. /** Given a list of essential true dofs and the parallel assembled matrix @a A, eliminate the true dofs from the matrix, storing the eliminated @@ -169,6 +190,28 @@ public: HypreParMatrix &A) const { return A.EliminateRowsCols(tdofs_list); } + /// Eliminate essential true DOFs from the parallel system matrix. + /** Given a list of essential true dofs, eliminate the true dofs from + the parallel assembled system matrix, storing the eliminated part + internally. This method works in conjunction with + ParallelEliminateTDofsInRHS() and allows elimination of boundary + conditions in multiple right-hand sides. */ + void ParallelEliminateTDofs(const Array &tdofs_list); + + /** @brief Use the stored eliminated part of the parallel system matrix for + elimination of boundary conditions in the r.h.s. */ + /** Given a list of essential true dofs, eliminate the true dofs from the + right-hand side @a b using the solution vector @a x and the previously + stored eliminated part of the parallel assembled system matrix produced + by ParallelEliminateTDofs(const Array &). */ + void ParallelEliminateTDofsInRHS(const Array &tdofs, const Vector &x, + Vector &b); + + /// @deprecated Use ParallelEliminateTDofsInRHS() instead. + MFEM_DEPRECATED void EliminateVDofsInRHS(const Array &vdofs, + const Vector &x, Vector &b) + { ParallelEliminateTDofsInRHS(vdofs, x, b); } + /** @brief Compute @a y += @a a (P^t A P) @a x, where @a x and @a y are vectors on the true dofs. */ void TrueAddMult(const Vector &x, Vector &y, const real_t a = 1.0) const; @@ -238,8 +281,6 @@ public: void Update(FiniteElementSpace *nfes = NULL) override; - void EliminateVDofsInRHS(const Array &vdofs, const Vector &x, Vector &b); - virtual ~ParBilinearForm() { } }; @@ -257,6 +298,13 @@ protected: /// Matrix and eliminated matrix OperatorHandle p_mat, p_mat_e; + bool keep_nbr_block; + + // Allocate mat - called when (mat == NULL && fbfi.Size() > 0) + void pAllocMat(); + + void AssembleSharedFaces(int skip_zeros = 1); + private: /// Copy construction is not supported; body is undefined. ParMixedBilinearForm(const ParMixedBilinearForm &); @@ -276,6 +324,7 @@ public: { trial_pfes = trial_fes; test_pfes = test_fes; + keep_nbr_block = false; } /** @brief Create a ParMixedBilinearForm on the given FiniteElementSpace%s @@ -295,15 +344,89 @@ public: { trial_pfes = trial_fes; test_pfes = test_fes; + keep_nbr_block = false; } + /** When set to true and the ParMixedBilinearForm has interior face + integrators, the local SparseMatrix will include the rows (in addition + to the columns) corresponding to face-neighbor dofs. The default + behavior is to disregard those rows. Must be called before the first + Assemble() call. */ + void KeepNbrBlock(bool knb = true) { keep_nbr_block = knb; } + + /// Assemble the local matrix + void Assemble(int skip_zeros = 1); + /// Returns the matrix assembled on the true dofs, i.e. P_test^t A P_trial. - HypreParMatrix *ParallelAssemble(); + /** The returned matrix is the internal one, owned by the form. It is not + reassembled if it has been already constructed. If + FormRectangularSystemMatrix() has been called before, it is the system + matrix with eliminated essential DOFs, otherwise the parallel matrix is + assembled here without the elimination process. */ + HypreParMatrix *ParallelAssembleInternalMatrix(); + + /// Returns the matrix assembled on the true dofs, i.e. P_test^t A P_trial. + /** The returned matrix has to be deleted by the caller. */ + HypreParMatrix *ParallelAssemble() { return ParallelAssemble(mat); } + + /** @brief Returns the eliminated matrix assembled on the true dofs, i.e. + P_test^t A_local P_trial. */ + /** The returned matrix has to be deleted by the caller. */ + HypreParMatrix *ParallelAssembleElim() { return ParallelAssemble(mat_e); } + + /** @brief Return the matrix @a m assembled on the true dofs, i.e. P_test^t + A_local P_trial. */ + /** The returned matrix has to be deleted by the caller. */ + HypreParMatrix *ParallelAssemble(SparseMatrix *m); /** @brief Returns the matrix assembled on the true dofs, i.e. @a A = P_test^t A_local P_trial, in the format (type id) specified by @a A. */ - void ParallelAssemble(OperatorHandle &A); + void ParallelAssemble(OperatorHandle &A) { ParallelAssemble(A, mat); } + + /** Returns the eliminated matrix assembled on the true dofs, i.e. + @a A_elim = P^t A_elim_local P in the format (type id) specified by @a A. + */ + void ParallelAssembleElim(OperatorHandle &A_elim) + { ParallelAssemble(A_elim, mat_e); } + + /** Returns the matrix @a A_local assembled on the true dofs, i.e. + @a A = P_test^t A_local P_trial in the format (type id) specified by + @a A. */ + void ParallelAssemble(OperatorHandle &A, SparseMatrix *A_local); + + /// Eliminate essential boundary trial DOFs from the parallel system matrix. + /** The array @a bdr_attr_is_ess marks boundary attributes that constitute + the essential part of the boundary. This method relies on + ParallelEliminateTrialTDofs(const Array &), see it for details. */ + void ParallelEliminateTrialEssentialBC(const Array &bdr_attr_is_ess); + + /// Eliminate essential trial true DOFs from the parallel system matrix. + /** Given a list of essential trial true dofs, eliminate the trial true dofs + from the parallel assembled system matrix, storing the eliminated part + internally. This method works in conjunction with + ParallelEliminateTrialTDofsInRHS() and allows elimination of boundary + conditions in multiple right-hand sides. */ + void ParallelEliminateTrialTDofs(const Array &trial_tdof_list); + + /** @brief Use the stored eliminated part of the parallel system matrix for + elimination of boundary conditions in the r.h.s. */ + /** Given a list of essential trial true dofs, eliminate the trial true dofs + from the right-hand side @a B using the solution vector @a X and the + previously stored eliminated part of the parallel assembled system + matrix produced by ParallelEliminateTrialTDofs(const Array &). */ + void ParallelEliminateTrialTDofsInRHS(const Array &trial_tdof_list, + const Vector &X, Vector &B); + + /// Eliminate essential boundary test DOFs from the parallel system matrix. + /** The array @a bdr_attr_is_ess marks boundary attributes that constitute + the essential part of the boundary. */ + void ParallelEliminateTestEssentialBC(const Array &bdr_attr_is_ess); + + /// Eliminate essential test true DOFs from the parallel system matrix. + /** Given a list of essential test true dofs, eliminate the test true dofs + from the parallel assembled system matrix. */ + void ParallelEliminateTestTDofs(const Array &test_tdof_list); using MixedBilinearForm::FormRectangularSystemMatrix; using MixedBilinearForm::FormRectangularLinearSystem; diff --git a/fem/pderefmat_op.cpp b/fem/pderefmat_op.cpp index 54c97ec640..27f0d84a0f 100644 --- a/fem/pderefmat_op.cpp +++ b/fem/pderefmat_op.cpp @@ -332,7 +332,7 @@ ParDerefineMatrixOp::ParDerefineMatrixOp(ParFiniteElementSpace &fespace_, pack_col_idcs.SetSize(send_len); // memory manager doesn't appear to have a graceful fallback for // HOST_PINNED if not built with CUDA or HIP -#if defined(MFEM_USE_CUDA) or defined(MFEM_USE_HIP) +#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) xghost_send.SetSize(send_len * fespace->GetVDim(), Device::GetGPUAwareMPI() ? MemoryType::DEFAULT : MemoryType::HOST_PINNED); diff --git a/fem/pfespace.hpp b/fem/pfespace.hpp index 90c06e1beb..e190e43d1c 100644 --- a/fem/pfespace.hpp +++ b/fem/pfespace.hpp @@ -481,6 +481,7 @@ public: that the number of DOFs is @a ndofs. */ const FiniteElement *GetFaceNbrFE(int i, int ndofs = 0) const; const FiniteElement *GetFaceNbrFaceFE(int i) const; + const Array &GetFaceNbrGlobalDofMapArray() { return face_nbr_glob_dof_map; } const HYPRE_BigInt *GetFaceNbrGlobalDofMap() { return face_nbr_glob_dof_map; } ElementTransformation *GetFaceNbrElementTransformation(int i) const { return pmesh->GetFaceNbrElementTransformation(i); } diff --git a/fem/pnonlinearform.cpp b/fem/pnonlinearform.cpp index d345145165..4538742146 100644 --- a/fem/pnonlinearform.cpp +++ b/fem/pnonlinearform.cpp @@ -105,6 +105,59 @@ const SparseMatrix &ParNonlinearForm::GetLocalGradient(const Vector &x) const return *Grad; } +void ParNonlinearForm::GradientSharedFaces(const Vector &x, + int skip_zeros) const +{ + ParFiniteElementSpace *pfes = ParFESpace(); + ParMesh *pmesh = pfes->GetParMesh(); + FaceElementTransformations *T; + Array vdofs1, vdofs2, vdofs_all; + DenseMatrix elemmat; + Vector el_x, nbr_x, face_x; + const Vector &px = Prolongate(x); + + ParGridFunction pgf(pfes, const_cast(px), 0); + pgf.ExchangeFaceNbrData(); + + int nfaces = pmesh->GetNSharedFaces(); + for (int i = 0; i < nfaces; i++) + { + T = pmesh->GetSharedFaceTransformations(i); + int Elem2NbrNo = T->Elem2No - pmesh->GetNE(); + + pfes->GetElementVDofs(T->Elem1No, vdofs1); + pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2); + face_x.SetSize(vdofs1.Size() + vdofs2.Size()); + + el_x.MakeRef(face_x, 0, vdofs1.Size()); + pgf.GetSubVector(vdofs1, el_x); + + nbr_x.MakeRef(face_x, vdofs1.Size(), vdofs2.Size()); + pgf.FaceNbrData().GetSubVector(vdofs2, nbr_x); + + vdofs1.Copy(vdofs_all); + for (int j = 0; j < vdofs2.Size(); j++) + { + if (vdofs2[j] >= 0) + { + vdofs2[j] += height; + } + else + { + vdofs2[j] -= height; + } + } + vdofs_all.Append(vdofs2); + for (int k = 0; k < fnfi.Size(); k++) + { + fnfi[k]->AssembleFaceGrad(*pfes->GetFE(T->Elem1No), + *pfes->GetFaceNbrFE(Elem2NbrNo), + *T, face_x, elemmat); + Grad->AddSubMatrix(vdofs1, vdofs_all, elemmat, skip_zeros); + } + } +} + Operator &ParNonlinearForm::GetGradient(const Vector &x) const { if (NonlinearForm::ext) { return NonlinearForm::GetGradient(x); } @@ -112,19 +165,61 @@ Operator &ParNonlinearForm::GetGradient(const Vector &x) const ParFiniteElementSpace *pfes = ParFESpace(); pGrad.Clear(); + OperatorHandle dA(pGrad.Type()), Ph(pGrad.Type()), hdA; - NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c. - - OperatorHandle dA(pGrad.Type()), Ph(pGrad.Type()); - - if (fnfi.Size() == 0) + if (fnfi.Size()) { - dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(), - pfes->GetDofOffsets(), Grad); + const int skip_zeros = 0; + + pfes->ExchangeFaceNbrData(); + if (Grad == NULL) + { + int nbr_size = pfes->GetFaceNbrVSize(); + Grad = new SparseMatrix(pfes->GetVSize(), pfes->GetVSize() + nbr_size); + } + + NonlinearForm::GetGradient(x, false); // (re)assemble Grad, no b.c. + + GradientSharedFaces(x, skip_zeros); + + Grad->Finalize(skip_zeros); + + // handle the case when 'a' contains off-diagonal + int lvsize = pfes->GetVSize(); + const HYPRE_BigInt *face_nbr_glob_ldof = pfes->GetFaceNbrGlobalDofMap(); + HYPRE_BigInt ldof_offset = pfes->GetMyDofOffset(); + + Array glob_J(Grad->NumNonZeroElems()); + int *J = Grad->GetJ(); + for (int i = 0; i < glob_J.Size(); i++) + { + if (J[i] < lvsize) + { + glob_J[i] = J[i] + ldof_offset; + } + else + { + glob_J[i] = face_nbr_glob_ldof[J[i] - lvsize]; + } + } + + // TODO - construct dA directly in the A format + hdA.Reset( + new HypreParMatrix(pfes->GetComm(), lvsize, pfes->GlobalVSize(), + pfes->GlobalVSize(), Grad->GetI(), glob_J, + Grad->GetData(), pfes->GetDofOffsets(), + pfes->GetDofOffsets())); + // - hdA owns the new HypreParMatrix + // - the above constructor copies all input arrays + glob_J.DeleteAll(); + dA.ConvertFrom(hdA); } else { - MFEM_ABORT("TODO: assemble contributions from shared face terms"); + NonlinearForm::GetGradient(x); // (re)assemble Grad, no b.c. + + dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(), + pfes->GetDofOffsets(), Grad); } // RAP the local gradient dA. @@ -199,9 +294,8 @@ const ParFiniteElementSpace *ParBlockNonlinearForm::ParFESpace(int k) const } // Here, rhs is a true dof vector -void ParBlockNonlinearForm::SetEssentialBC(const - Array *>&bdr_attr_is_ess, - Array &rhs) +void ParBlockNonlinearForm::SetEssentialBC( + const Array*> &bdr_attr_is_ess, Array &rhs) { Array nullarray(fes.Size()); nullarray = NULL; @@ -217,6 +311,23 @@ void ParBlockNonlinearForm::SetEssentialBC(const } } +void ParBlockNonlinearForm::SetEssentialTrueDofs( + const Array*> &ess_tdof_list, Array &rhs) +{ + Array nullarray(fes.Size()); + nullarray = nullptr; + + BlockNonlinearForm::SetEssentialTrueDofs(ess_tdof_list, nullarray); + + for (int s = 0; s < fes.Size(); ++s) + { + if (rhs[s]) + { + rhs[s]->SetSubVector(*ess_tdofs[s], 0.0); + } + } +} + real_t ParBlockNonlinearForm::GetEnergy(const Vector &x) const { // xs_true is not modified, so const_cast is okay @@ -255,7 +366,70 @@ void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const if (fnfi.Size() > 0) { - MFEM_ABORT("TODO: assemble contributions from shared face terms"); + // Terms over shared interior faces in parallel. + ParMesh *pmesh = ParFESpace(0)->GetParMesh(); + FaceElementTransformations *tr; + + Array *>vdofs(fes.Size()); + Array *>vdofs2(fes.Size()); + Array el_x(fes.Size()); + Array el_x_const(fes.Size()); + Array el_y(fes.Size()); + Array fe(fes.Size()); + Array fe2(fes.Size()); + Array pgfs(fes.Size()); + for (int s=0; s; + vdofs2[s] = new Array; + pgfs[s] = new ParGridFunction(const_cast(ParFESpace(s)), + xs.GetBlock(s)); + pgfs[s]->ExchangeFaceNbrData(); + } + + const int n_shared_faces = pmesh->GetNSharedFaces(); + for (int i = 0; i < n_shared_faces; i++) + { + tr = pmesh->GetSharedFaceTransformations(i, true); + int Elem2NbrNo = tr->Elem2No - pmesh->GetNE(); + + for (int s=0; sGetFE(tr->Elem1No); + fe2[s] = pfes->GetFaceNbrFE(Elem2NbrNo); + + pfes->GetElementVDofs(tr->Elem1No, *(vdofs[s])); + pfes->GetFaceNbrElementVDofs(Elem2NbrNo, *(vdofs2[s])); + + el_x[s]->SetSize(vdofs[s]->Size() + vdofs2[s]->Size()); + xs.GetBlock(s).GetSubVector(*(vdofs[s]), el_x[s]->GetData()); + pgfs[s]->FaceNbrData().GetSubVector(*(vdofs2[s]), + el_x[s]->GetData() + vdofs[s]->Size()); + } + + for (int k = 0; k < fnfi.Size(); ++k) + { + fnfi[k]->AssembleFaceVector(fe, fe2, *tr, el_x_const, el_y); + + for (int s=0; sSize() == 0) { continue; } + ys.GetBlock(s).AddElementVector(*(vdofs[s]), *el_y[s]); + } + } + } + + for (int s=0; sGetParMesh(); + FaceElementTransformations *tr; + + Array *>vdofs(fes.Size()); + Array *>vdofs2(fes.Size()); + Array *>vdofs_all(fes.Size()); + Array el_x(fes.Size()); + Array el_x_const(fes.Size()); + Array2D elmats(fes.Size(), fes.Size()); + Array fe(fes.Size()); + Array fe2(fes.Size()); + Array pgfs(fes.Size()); + + for (int s1=0; s1; + vdofs2[s1] = new Array; + vdofs_all[s1] = new Array; + pgfs[s1] = new ParGridFunction( + const_cast(ParFESpace(s1)), + const_cast(xs.GetBlock(s1))); + pgfs[s1]->ExchangeFaceNbrData(); + for (int s2=0; s2GetNSharedFaces(); + for (int i = 0; i < n_shared_faces; i++) + { + tr = pmesh->GetSharedFaceTransformations(i, true); + int Elem2NbrNo = tr->Elem2No - pmesh->GetNE(); + + for (int s=0; sGetFE(tr->Elem1No); + fe2[s] = pfes->GetFaceNbrFE(Elem2NbrNo); + + pfes->GetElementVDofs(tr->Elem1No, *(vdofs[s])); + pfes->GetFaceNbrElementVDofs(Elem2NbrNo, *(vdofs2[s])); + + el_x[s]->SetSize(vdofs[s]->Size() + vdofs2[s]->Size()); + xs.GetBlock(s).GetSubVector(*(vdofs[s]), el_x[s]->GetData()); + pgfs[s]->FaceNbrData().GetSubVector(*(vdofs2[s]), + el_x[s]->GetData() + vdofs[s]->Size()); + + vdofs[s]->Copy(*vdofs_all[s]); + + const int lvsize = pfes->GetVSize(); + for (int j = 0; j < vdofs2[s]->Size(); j++) + { + if ((*vdofs2[s])[j] >= 0) + { + (*vdofs2[s])[j] += lvsize; + } + else + { + (*vdofs2[s])[j] -= lvsize; + } + } + vdofs_all[s]->Append(*(vdofs2[s])); + } + + for (int k = 0; k < fnfi.Size(); ++k) + { + fnfi[k]->AssembleFaceGrad(fe, fe2, *tr, el_x_const, elmats); + + for (int s1=0; s1Height() == 0) { continue; } + Grads(s1,s2)->AddSubMatrix(*vdofs[s1], *vdofs_all[s2], + *elmats(s1,s2), skip_zeros); + } + } + } + } + + for (int s1=0; s1(x), block_trueOffsets); + xs.Update(block_offsets); + + for (int s=0; sGetProlongationMatrix()->Mult( + xs_true.GetBlock(s), xs.GetBlock(s)); + } if (fnfi.Size() > 0) { - MFEM_ABORT("TODO: assemble contributions from shared face terms"); - } + const int skip_zeros = 0; - for (int s1=0; s1Type()), - Ph(phBlockGrad(s1,s2)->Type()), - Rh(phBlockGrad(s1,s2)->Type()); + const_cast(pfes[s])->ExchangeFaceNbrData(); + } - if (s1 == s2) + for (int s1=0; s1GetComm(), pfes[s1]->GlobalVSize(), - pfes[s1]->GetDofOffsets(), Grads(s1,s1)); - Ph.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix()); - phBlockGrad(s1,s1)->MakePtAP(dA, Ph); - - OperatorHandle Ae; - Ae.EliminateRowsCols(*phBlockGrad(s1,s1), *ess_tdofs[s1]); + if (Grads(s1,s2) == NULL) + { + int nbr_size = pfes[s2]->GetFaceNbrVSize(); + Grads(s1,s2) = new SparseMatrix(pfes[s1]->GetVSize(), + pfes[s2]->GetVSize() + nbr_size); + } } - else + } + + // (re)assemble Grad without b.c. into 'Grads' + BlockNonlinearForm::ComputeGradientBlocked(xs, false); + + GradientSharedFaces(xs, skip_zeros); + + // finalize the gradients + for (int s1=0; s1GetComm(), - pfes[s1]->GlobalVSize(), - pfes[s2]->GlobalVSize(), - pfes[s1]->GetDofOffsets(), - pfes[s2]->GetDofOffsets(), - Grads(s1,s2)); - Rh.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix()); - Ph.ConvertFrom(pfes[s2]->Dof_TrueDof_Matrix()); - - phBlockGrad(s1,s2)->MakeRAP(Rh, dA, Ph); - - phBlockGrad(s1,s2)->EliminateRows(*ess_tdofs[s1]); - phBlockGrad(s1,s2)->EliminateCols(*ess_tdofs[s2]); + Grads(s1,s2)->Finalize(skip_zeros); } - pBlockGrad->SetBlock(s1, s2, phBlockGrad(s1,s2)->Ptr()); + for (int s1=0; s1Type()), + Ph(phBlockGrad(s1,s2)->Type()), + Rh(phBlockGrad(s1,s2)->Type()); + + // handle the case when 'a' contains off-diagonal + int lvsize = pfes[s2]->GetVSize(); + const HYPRE_BigInt *face_nbr_glob_ldof = + const_cast(pfes[s2])->GetFaceNbrGlobalDofMap(); + HYPRE_BigInt ldof_offset = pfes[s2]->GetMyDofOffset(); + + Array glob_J(Grads(s1,s2)->NumNonZeroElems()); + int *J = Grads(s1,s2)->GetJ(); + for (int i = 0; i < glob_J.Size(); i++) + { + if (J[i] < lvsize) + { + glob_J[i] = J[i] + ldof_offset; + } + else + { + glob_J[i] = face_nbr_glob_ldof[J[i] - lvsize]; + } + } + + // TODO - construct dA directly in the A format + hdA.Reset( + new HypreParMatrix(pfes[s2]->GetComm(), pfes[s1]->GetVSize(), + pfes[s1]->GlobalVSize(), pfes[s2]->GlobalVSize(), + Grads(s1,s2)->GetI(), glob_J, Grads(s1,s2)->GetData(), + pfes[s1]->GetDofOffsets(), pfes[s2]->GetDofOffsets())); + // - hdA owns the new HypreParMatrix + // - the above constructor copies all input arrays + glob_J.DeleteAll(); + dA.ConvertFrom(hdA); + + if (s1 == s2) + { + Ph.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix()); + phBlockGrad(s1,s1)->MakePtAP(dA, Ph); + + OperatorHandle Ae; + Ae.EliminateRowsCols(*phBlockGrad(s1,s1), *ess_tdofs[s1]); + } + else + { + Rh.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix()); + Ph.ConvertFrom(pfes[s2]->Dof_TrueDof_Matrix()); + + phBlockGrad(s1,s2)->MakeRAP(Rh, dA, Ph); + + phBlockGrad(s1,s2)->EliminateRows(*ess_tdofs[s1]); + phBlockGrad(s1,s2)->EliminateCols(*ess_tdofs[s2]); + } + + pBlockGrad->SetBlock(s1, s2, phBlockGrad(s1,s2)->Ptr()); + } + } + } + else + { + // (re)assemble Grad without b.c. into 'Grads' + BlockNonlinearForm::ComputeGradientBlocked(xs); + + for (int s1=0; s1Type()), + Ph(phBlockGrad(s1,s2)->Type()), + Rh(phBlockGrad(s1,s2)->Type()); + + if (s1 == s2) + { + dA.MakeSquareBlockDiag(pfes[s1]->GetComm(), pfes[s1]->GlobalVSize(), + pfes[s1]->GetDofOffsets(), Grads(s1,s1)); + Ph.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix()); + phBlockGrad(s1,s1)->MakePtAP(dA, Ph); + + OperatorHandle Ae; + Ae.EliminateRowsCols(*phBlockGrad(s1,s1), *ess_tdofs[s1]); + } + else + { + dA.MakeRectangularBlockDiag(pfes[s1]->GetComm(), + pfes[s1]->GlobalVSize(), + pfes[s2]->GlobalVSize(), + pfes[s1]->GetDofOffsets(), + pfes[s2]->GetDofOffsets(), + Grads(s1,s2)); + Rh.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix()); + Ph.ConvertFrom(pfes[s2]->Dof_TrueDof_Matrix()); + + phBlockGrad(s1,s2)->MakeRAP(Rh, dA, Ph); + + phBlockGrad(s1,s2)->EliminateRows(*ess_tdofs[s1]); + phBlockGrad(s1,s2)->EliminateCols(*ess_tdofs[s2]); + } + + pBlockGrad->SetBlock(s1, s2, phBlockGrad(s1,s2)->Ptr()); + } } } diff --git a/fem/pnonlinearform.hpp b/fem/pnonlinearform.hpp index f4a567dfbb..112c6c9d57 100644 --- a/fem/pnonlinearform.hpp +++ b/fem/pnonlinearform.hpp @@ -29,6 +29,8 @@ protected: mutable ParGridFunction X, Y; mutable OperatorHandle pGrad; + void GradientSharedFaces(const Vector &x, int skip_zeros = 1) const; + public: ParNonlinearForm(ParFiniteElementSpace *pf); @@ -81,6 +83,8 @@ protected: mutable Array2D phBlockGrad; mutable BlockOperator *pBlockGrad; + void GradientSharedFaces(const BlockVector &xs, int skip_zeros) const; + public: /// Computes the energy of the system real_t GetEnergy(const Vector &x) const override; @@ -102,9 +106,39 @@ public: gradient-type (if different from the default) must be set again. */ void SetParSpaces(Array &pf); - // Here, rhs is a true dof vector - void SetEssentialBC(const Array *>&bdr_attr_is_ess, - Array &rhs) override; + /** @brief Set essential boundary conditions to each finite element space + using boundary attribute markers. + + This method calls `FiniteElementSpace::GetEssentialTrueDofs()` for each + space and stores ess_tdof_lists internally. + + If `rhs` vectors are non-null, the entries corresponding to these + essential DoFs are set to zero. This ensures compatibility with the + output of the `Mult()` method, which also zeroes out these entries. + + @param[in] bdr_attr_is_ess A list of boundary attribute markers for each + space. + @param[in,out] rhs An array of optional right-hand side vectors. + If a vector at `rhs[i]` is non-null, its essential DoFs will be set + to zero. */ + virtual void SetEssentialBC(const Array*> &bdr_attr_is_ess, + Array &rhs) override; + + /** @brief Set essential boundary conditions to each finite element space + using essential true dof lists. + + This method stores a copy of the provided essential true dof lists. + + If `rhs` vectors are non-null, the entries corresponding to these + essential DoFs are set to zero. This ensures compatibility with the + output of the `Mult()` method, which also zeroes out these entries. + + @param[in] ess_tdof_list A list of essential true dofs for each space. + @param[in,out] rhs An array of optional right-hand side vectors. + If a vector at `rhs[i]` is non-null, its essential DoFs will be set + to zero. */ + virtual void SetEssentialTrueDofs(const Array*> &ess_tdof_list, + Array &rhs) override; /// Block T-Vector to Block T-Vector void Mult(const Vector &x, Vector &y) const override; diff --git a/fem/qfunction.hpp b/fem/qfunction.hpp index 4feabe0b16..1a62c06540 100644 --- a/fem/qfunction.hpp +++ b/fem/qfunction.hpp @@ -41,6 +41,12 @@ public: qspace(&qspace_), own_qspace(false), vdim(vdim_) { UseDevice(true); } + /// Same as above but specify the device memory type + QuadratureFunction(QuadratureSpaceBase &qspace_, MemoryType mt, int vdim_ = 1) + : Vector(vdim_*qspace_.GetSize(), mt), + qspace(&qspace_), own_qspace(false), vdim(vdim_) + { UseDevice(true); } + /// Create a QuadratureFunction based on the given QuadratureSpaceBase. /** The QuadratureFunction does not assume ownership of the QuadratureSpaceBase. @@ -48,6 +54,10 @@ public: QuadratureFunction(QuadratureSpaceBase *qspace_, int vdim_ = 1) : QuadratureFunction(*qspace_, vdim_) { } + /// Same as above but specify the device memory type + QuadratureFunction(QuadratureSpaceBase *qspace_, MemoryType mt, int vdim_ = 1) + : QuadratureFunction(*qspace_, mt, vdim_) { } + /** @brief Create a QuadratureFunction based on the given QuadratureSpaceBase, using the external (host) data, @a qf_data. */ /** The QuadratureFunction does not assume ownership of the @@ -214,16 +224,16 @@ public: inline void QuadratureFunction::GetValues( int idx, Vector &values) { - const int s_offset = qspace->offsets[idx]; - const int sl_size = qspace->offsets[idx+1] - s_offset; + const int s_offset = qspace->Offset(idx); + const int sl_size = qspace->Offset(idx + 1) - s_offset; values.MakeRef(*this, vdim*s_offset, vdim*sl_size); } inline void QuadratureFunction::GetValues( int idx, Vector &values) const { - const int s_offset = qspace->offsets[idx]; - const int sl_size = qspace->offsets[idx+1] - s_offset; + const int s_offset = qspace->Offset(idx); + const int sl_size = qspace->Offset(idx + 1) - s_offset; values.SetSize(vdim*sl_size); values.HostWrite(); const real_t *q = HostRead() + vdim*s_offset; @@ -236,14 +246,14 @@ inline void QuadratureFunction::GetValues( inline void QuadratureFunction::GetValues( int idx, const int ip_num, Vector &values) { - const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim; + const int s_offset = qspace->Offset(idx) * vdim + ip_num * vdim; values.MakeRef(*this, s_offset, vdim); } inline void QuadratureFunction::GetValues( int idx, const int ip_num, Vector &values) const { - const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim; + const int s_offset = qspace->Offset(idx) * vdim + ip_num * vdim; values.SetSize(vdim); values.HostWrite(); const real_t *q = HostRead() + s_offset; @@ -256,8 +266,8 @@ inline void QuadratureFunction::GetValues( inline void QuadratureFunction::GetValues( int idx, DenseMatrix &values) { - const int s_offset = qspace->offsets[idx]; - const int sl_size = qspace->offsets[idx+1] - s_offset; + const int s_offset = qspace->Offset(idx); + const int sl_size = qspace->Offset(idx + 1) - s_offset; // Make the values matrix memory an alias of the quadrature function memory Memory &values_mem = values.GetMemory(); values_mem.Delete(); @@ -268,8 +278,8 @@ inline void QuadratureFunction::GetValues( inline void QuadratureFunction::GetValues( int idx, DenseMatrix &values) const { - const int s_offset = qspace->offsets[idx]; - const int sl_size = qspace->offsets[idx+1] - s_offset; + const int s_offset = qspace->Offset(idx); + const int sl_size = qspace->Offset(idx + 1) - s_offset; values.SetSize(vdim, sl_size); values.HostWrite(); const real_t *q = HostRead() + vdim*s_offset; diff --git a/fem/qinterp/det.cpp b/fem/qinterp/det.cpp index 4ed8f2fd5a..304bfb8561 100644 --- a/fem/qinterp/det.cpp +++ b/fem/qinterp/det.cpp @@ -37,6 +37,8 @@ void InitDetKernels() k::Specialization<3,3,3,3>::Add(); k::Specialization<3,3,3,5>::Add(); k::Specialization<3,3,3,6>::Add(); + k::Specialization<3,3,4,6>::Add(); + k::Specialization<3,3,3,4>::Add(); } } // namespace quadrature_interpolator diff --git a/fem/qinterp/eval_by_nodes.cpp b/fem/qinterp/eval_by_nodes.cpp index eae30b523b..bfea76d815 100644 --- a/fem/qinterp/eval_by_nodes.cpp +++ b/fem/qinterp/eval_by_nodes.cpp @@ -28,6 +28,7 @@ void InitEvalByNodesKernels() k::Specialization<2,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add(); k::Specialization<2,QVectorLayout::byNODES,1,3,2>::Opt<1>::Add(); k::Specialization<2,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add(); + k::Specialization<2,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add(); k::Specialization<2,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add(); k::Specialization<2,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add(); diff --git a/fem/qinterp/eval_by_vdim.cpp b/fem/qinterp/eval_by_vdim.cpp index 3ecb8fa17d..9dd685fe13 100644 --- a/fem/qinterp/eval_by_vdim.cpp +++ b/fem/qinterp/eval_by_vdim.cpp @@ -30,6 +30,7 @@ void InitEvalByVDimKernels() k::Specialization<2,QVectorLayout::byVDIM,2,2,4>::Opt<8>::Add(); k::Specialization<2,QVectorLayout::byVDIM,2,3,4>::Opt<8>::Add(); k::Specialization<2,QVectorLayout::byVDIM,2,3,6>::Opt<4>::Add(); + k::Specialization<2,QVectorLayout::byVDIM,2,4,6>::Opt<2>::Add(); k::Specialization<2,QVectorLayout::byVDIM,2,4,8>::Opt<2>::Add(); // 3D k::Specialization<3,QVectorLayout::byVDIM,1,2,4>::Opt<1>::Add(); @@ -47,6 +48,9 @@ void InitEvalByVDimKernels() k::Specialization<3,QVectorLayout::byVDIM,3,7,7>::Opt<1>::Add(); k::Specialization<3,QVectorLayout::byVDIM,3,8,8>::Opt<1>::Add(); k::Specialization<3,QVectorLayout::byVDIM,3,9,9>::Opt<1>::Add(); + + k::Specialization<3,QVectorLayout::byVDIM,3,4,6>::Opt<1>::Add(); + k::Specialization<3,QVectorLayout::byVDIM,3,3,4>::Opt<1>::Add(); } } // namespace quadrature_interpolator diff --git a/fem/qspace.cpp b/fem/qspace.cpp index 0e0b34a8ef..beff0cc569 100644 --- a/fem/qspace.cpp +++ b/fem/qspace.cpp @@ -22,7 +22,7 @@ QuadratureSpaceBase::QuadratureSpaceBase(Mesh &mesh_, Geometry::Type geom, { for (int g = 0; g < Geometry::NumGeom; g++) { - int_rule[g] = NULL; + int_rule[g] = nullptr; } int_rule[geom] = &ir; } @@ -37,6 +37,29 @@ void QuadratureSpaceBase::ConstructIntRules(int dim) } } +const Array &QuadratureSpaceBase::Offsets( + QSpaceOffsetStorage storage) const +{ + if (storage == QSpaceOffsetStorage::COMPRESSED || offsets.Size() > 1) + { + return offsets; + } + else + { + if (full_offset_cache.Size() == 0) + { + const int nq = size / ne; + full_offset_cache.SetSize(ne + 1); + int *d_full_offset_cache = full_offset_cache.Write(); + mfem::forall(ne + 1, [=] MFEM_HOST_DEVICE (int e) + { + d_full_offset_cache[e] = nq * e; + }); + } + return full_offset_cache; + } +} + namespace { @@ -97,16 +120,31 @@ void QuadratureSpaceBase::Integrate(VectorCoefficient &coeff, void QuadratureSpace::ConstructOffsets() { const int num_elem = mesh.GetNE(); - offsets.SetSize(num_elem + 1); - int offset = 0; - for (int i = 0; i < num_elem; i++) + ne = num_elem; + + if (mesh.GetNumGeometries(mesh.Dimension()) == 1) { - offsets[i] = offset; - int geom = mesh.GetElementBaseGeometry(i); - MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule."); - offset += int_rule[geom]->GetNPoints(); + Array geoms; + mesh.GetGeometries(mesh.Dimension(), geoms); + offsets.SetSize(1); + offsets.HostWrite(); + offsets[0] = int_rule[geoms[0]]->GetNPoints(); + size = num_elem * offsets[0]; + } + else + { + offsets.SetSize(num_elem + 1); + int offset = 0; + for (int i = 0; i < num_elem; i++) + { + offsets[i] = offset; + const Geometry::Type geom = mesh.GetElementBaseGeometry(i); + MFEM_ASSERT(int_rule[geom] != nullptr, "Missing integration rule."); + offset += int_rule[geom]->GetNPoints(); + } + offsets[num_elem] = offset; + size = offsets.Last(); } - offsets[num_elem] = size = offset; } void QuadratureSpace::Construct() @@ -166,9 +204,9 @@ const Vector &QuadratureSpace::GetGeometricFactorWeights() const FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, int order_, FaceType face_type_) - : QuadratureSpaceBase(mesh_, order_), - face_type(face_type_), - num_faces(mesh.GetNFbyType(face_type)) + : QuadratureSpaceBase(mesh_, order_), face_type(face_type_), + face_indices(mesh.GetFaceIndices(face_type_)), + face_indices_inv(mesh.GetInvFaceIndices(face_type_)) { Construct(); } @@ -177,7 +215,8 @@ FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir, FaceType face_type_) : QuadratureSpaceBase(mesh_, mesh_.GetTypicalFaceGeometry(), ir), face_type(face_type_), - num_faces(mesh.GetNFbyType(face_type)) + face_indices(mesh.GetFaceIndices(face_type_)), + face_indices_inv(mesh.GetInvFaceIndices(face_type_)) { MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension() - 1) <= 1, "Constructor not valid for mixed meshes"); @@ -186,27 +225,30 @@ FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir, void FaceQuadratureSpace::ConstructOffsets() { - face_indices.SetSize(num_faces); - offsets.SetSize(num_faces + 1); - int offset = 0; - int f_idx = 0; - for (int i = 0; i < mesh.GetNumFacesWithGhost(); i++) - { - const Mesh::FaceInformation face = mesh.GetFaceInformation(i); - if (face.IsNonconformingCoarse() || !face.IsOfFaceType(face_type)) - { - continue; - } - face_indices[f_idx] = i; - face_indices_inv[i] = f_idx; - offsets[f_idx] = offset; - Geometry::Type geom = mesh.GetFaceGeometry(i); - MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule"); - offset += int_rule[geom]->GetNPoints(); + ne = face_indices.Size(); - f_idx++; + if (mesh.GetNumGeometries(mesh.Dimension() - 1) == 1) + { + Array geoms; + mesh.GetGeometries(mesh.Dimension() - 1, geoms); + offsets.SetSize(1); + offsets.HostWrite(); + offsets[0] = int_rule[geoms[0]]->GetNPoints(); + size = ne * offsets[0]; + } + else + { + offsets.SetSize(face_indices.Size() + 1); + int offset = 0; + for (int i = 0; i < mesh.GetNFbyType(face_type); ++i) + { + offsets[i] = offset; + Geometry::Type geom = mesh.GetFaceGeometry(face_indices[i]); + MFEM_ASSERT(int_rule[geom] != nullptr, "Missing integration rule"); + offset += int_rule[geom]->GetNPoints(); + } + offsets[face_indices.Size()] = size = offset; } - offsets[num_faces] = size = offset; } void FaceQuadratureSpace::Construct() @@ -232,6 +274,16 @@ int FaceQuadratureSpace::GetPermutedIndex(int idx, int iq) const } } +ElementTransformation *FaceQuadratureSpace::GetTransformation(int idx) +{ + ElementTransformation *T = mesh.GetFaceTransformation(face_indices[idx]); + if (face_type == FaceType::Boundary) + { + T->Attribute = mesh.GetBdrFaceAttributes()[idx]; + } + return T; +} + int FaceQuadratureSpace::GetEntityIndex(const ElementTransformation &T) const { auto get_face_index = [this](const int idx) diff --git a/fem/qspace.hpp b/fem/qspace.hpp index c9ab0a2082..df990488d7 100644 --- a/fem/qspace.hpp +++ b/fem/qspace.hpp @@ -19,6 +19,12 @@ namespace mfem { +enum class QSpaceOffsetStorage +{ + FULL, + COMPRESSED +}; + /// Abstract base class for QuadratureSpace and FaceQuadratureSpace. /** This class represents the storage layout for QuadratureFunction%s, that may be defined either on mesh elements or mesh faces. */ @@ -30,14 +36,28 @@ protected: Mesh &mesh; ///< The underlying mesh. int order; ///< The order of integration rule. int size; ///< Total number of quadrature points. + int ne; ///< Number of entities mutable Vector weights; ///< Integration weights. mutable long nodes_sequence = 0; ///< Nodes counter for cache invalidation. - /// @brief Entity quadrature point offset array, of size num_entities + 1. + /// @brief Entity quadrature point offset array. /// - /// The quadrature point values for entity i are stored in the indices between - /// offsets[i] and offsets[i+1]. + /// Supports a constant compression scheme for meshes which have a single + /// geometry type. When compressed, will have a single value. The true offset + /// can be computed as i * offsets[0], where i is the entity index. Otherwise + /// has size num_entities + 1. + /// + /// In the non-compressed case, the quadrature point values for entity i are + /// stored in the indices between offsets[i] and offsets[i+1]. Array offsets; + + /// @brief Cached version of the "full" offsets, returned by Offsets() when + /// QSpaceOffsetStorage::FULL is provided. + /// + /// The quadrature point values for entity i are stored in the indices + /// between offsets[i] and offsets[i+1]. + mutable Array full_offset_cache; + /// The quadrature rules used for each geometry type. const IntegrationRule *int_rule[Geometry::NumGeom]; @@ -59,6 +79,30 @@ protected: void ConstructWeights() const; public: + /// @brief Gets the offset for a given entity @a idx. + /// + /// The quadrature point values for entity i are stored in the indices + /// between Offset(i) and Offset(i+1) + int Offset(int idx) const + { + return (offsets.Size() == 1) ? (idx * offsets[0]) : offsets[idx]; + } + + /// @brief Entity quadrature point offset array. + /// + /// If @a storage is QSpaceOffsetStorage::COMPRESSED, then the returned array + /// supports a constant compression scheme for meshes which have a single + /// geometry type. When compressed, will have a single value. The true offset + /// can be computed as i * offsets[0], where i is the entity index. Otherwise + /// has size num_entities + 1. + /// + /// If @a storage is QSpaceOffsetStorage::FULL, then the array will never be + /// compressed. + /// + /// In the non-compressed case, the quadrature point values for entity i are + /// stored in the indices between offsets[i] and offsets[i+1]. + const Array &Offsets(QSpaceOffsetStorage storage) const; + /// Return the total number of quadrature points. int GetSize() const { return size; } @@ -66,7 +110,7 @@ public: int GetOrder() const { return order; } /// Return the number of entities. - int GetNE() const { return offsets.Size() - 1; } + int GetNE() const { return ne; } /// Returns the mesh. inline Mesh *GetMesh() const { return &mesh; } @@ -169,13 +213,12 @@ public: class FaceQuadratureSpace : public QuadratureSpaceBase { FaceType face_type; ///< Is the space defined on interior or boundary faces? - const int num_faces; ///< Number of faces. /// Map from boundary or interior face indices to mesh face indices. - Array face_indices; + const Array &face_indices; /// Inverse of the map @a face_indices. - std::unordered_map face_indices_inv; + const std::unordered_map &face_indices_inv; const Vector &GetGeometricFactorWeights() const override; void ConstructOffsets(); @@ -191,14 +234,13 @@ public: FaceType face_type_); /// Returns number of faces in the mesh. - inline int GetNumFaces() const { return num_faces; } + inline int GetNumFaces() const { return face_indices.Size(); } /// Returns the face type (boundary or interior). FaceType GetFaceType() const { return face_type; } /// Returns the face transformation of face @a idx. - ElementTransformation *GetTransformation(int idx) override - { return mesh.GetFaceTransformation(face_indices[idx]); } + ElementTransformation *GetTransformation(int idx) override; /// Returns the geometry type of face @a idx. Geometry::Type GetGeometry(int idx) const override diff --git a/fem/tmop.cpp b/fem/tmop.cpp index ac7b7c86e6..f2b4ed4300 100644 --- a/fem/tmop.cpp +++ b/fem/tmop.cpp @@ -140,6 +140,36 @@ void add_3D(const scalartype &scalar, const std::vector &u, /* Metric definitions */ +// W = ||T||^2 - 2*det(T). +template +type mu4_ad(const std::vector &T, const std::vector &W) +{ + auto fnorm2 = fnorm2_2D(T); + auto det = det_2D(T); + return fnorm2 - 2*det; +}; + +// W = ||T-I||^2. +template +type mu14_ad(const std::vector &T, const std::vector &W) +{ + DenseMatrix Id(2,2); Id = 0.0; + Id(0,0) = 1; Id(1,1) = 1; + + std::vector Mat; + add_2D(real_t{-1.0}, T, &Id, Mat); + + return fnorm2_2D(Mat); +}; + +// W = (det(T)-1)^2. +template +type mu55_ad(const std::vector &T, const std::vector &W) +{ + auto det = det_2D(T); + return pow(det-1.0, 2.0); +}; + // W = |T-T'|^2, where T'= |T|*I/sqrt(2). template type mu85_ad(const std::vector &T, const std::vector &W) @@ -163,6 +193,63 @@ type mu98_ad(const std::vector &T, const std::vector &W) return fnorm2_2D(Mat)/det_2D(T); }; +template +type make_one_type() +{ + return 1.0; +} +// add specialization for AD1Type +template <> +AD1Type make_one_type() +{ + return AD1Type{1.0, 0.0}; +} +// add specialization for AD2Type +template <> +AD2Type make_one_type() +{ + return AD2Type{AD1Type{1.0, 0.0}, AD1Type{0.0, 0.0}}; +} + +using TWCUO = TMOP_WorstCaseUntangleOptimizer_Metric; +template +type wcuo_ad(type mu, + const std::vector &T, const std::vector &W, + real_t alpha, real_t min_detT, real_t detT_ep, + int exponent, real_t max_muT, real_t muT_ep, + TWCUO::BarrierType bt, + TWCUO::WorstCaseType wct) +{ + type one = make_one_type(); + type zero = 0.0*one; + type denom = one; + if (bt == TWCUO::BarrierType::Shifted) + { + auto val1 = alpha*min_detT-detT_ep < 0.0 ? + (alpha*min_detT-detT_ep)*one : + zero; + denom = 2.0*(det_2D(T)-val1); + } + else if (bt == TWCUO::BarrierType::Pseudo) + { + auto detT = det_2D(T); + denom = detT + sqrt(detT*detT + detT_ep*detT_ep); + } + mu = mu/denom; + + if (wct == TWCUO::WorstCaseType::PMean) + { + auto exp = exponent*one; + mu = pow(mu, exp); + } + else if (wct == TWCUO::WorstCaseType::Beta) + { + auto beta = (max_muT+muT_ep)*one; + mu = mu/(beta-mu); + } + return mu; +} + // W = 1/(tau^0.5) |T-I|^2. template type mu342_ad(const std::vector &T, const std::vector &W) @@ -421,7 +508,7 @@ void TMOP_QualityMetric::DefaultAssembleH(const DenseTensor &H, { for (int cc = 0; cc < dim; cc++) { - const double entry_rr_cc = Hrc(rr, cc); + const real_t entry_rr_cc = Hrc(rr, cc); for (int i = 0; i < dof; i++) { @@ -481,6 +568,30 @@ void TMOP_Combo_QualityMetric::EvalPW(const DenseMatrix &Jpt, } } +AD1Type TMOP_Combo_QualityMetric::EvalW_AD1(const std::vector &T, + const std::vector &W) +const +{ + AD1Type metric = {0., 0.}; + for (int i = 0; i < tmop_q_arr.Size(); i++) + { + metric += wt_arr[i]*tmop_q_arr[i]->EvalW_AD1(T, W); + } + return metric; +} + +AD2Type TMOP_Combo_QualityMetric::EvalW_AD2(const std::vector &T, + const std::vector &W) +const +{ + AD2Type metric = {{0., 0.},{0., 0.}}; + for (int i = 0; i < tmop_q_arr.Size(); i++) + { + metric += wt_arr[i]*tmop_q_arr[i]->EvalW_AD2(T, W); + } + return metric; +} + void TMOP_Combo_QualityMetric::AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS, const real_t weight, @@ -645,6 +756,64 @@ real_t TMOP_WorstCaseUntangleOptimizer_Metric::EvalWBarrier( return tmop_metric.EvalW(Jpt)/denominator; } +AD1Type TMOP_WorstCaseUntangleOptimizer_Metric::EvalW_AD1( + const std::vector &T, + const std::vector &W) const +{ + return wcuo_ad(tmop_metric.EvalW_AD1(T,W), T, W, alpha, min_detT, detT_ep, + exponent, max_muT, muT_ep, btype, wctype); +} + +AD2Type TMOP_WorstCaseUntangleOptimizer_Metric::EvalW_AD2( + const std::vector &T, + const std::vector &W) const +{ + return wcuo_ad(tmop_metric.EvalW_AD2(T,W), T, W, alpha, min_detT, detT_ep, + exponent, max_muT, muT_ep, btype, wctype); +} + +void TMOP_WorstCaseUntangleOptimizer_Metric::EvalP(const DenseMatrix &Jpt, + DenseMatrix &P) const +{ + auto mu_ad_fn = [this](std::vector &T, std::vector &W) + { + return EvalW_AD1(T,W); + }; + if (tmop_metric.Id() == 4 || tmop_metric.Id() == 14 || + tmop_metric.Id() == 66) + { + ADGrad(mu_ad_fn, P, Jpt); + return; + } + MFEM_ABORT("EvalW_AD1 not implemented with this metric for " + "TMOP_WorstCaseUntangleOptimizer_Metric. " + "Please use metric 4/14/66."); +} + +void TMOP_WorstCaseUntangleOptimizer_Metric::AssembleH( + const DenseMatrix &Jpt, + const DenseMatrix &DS, + const real_t weight, + DenseMatrix &A) const +{ + DenseTensor H(Jpt.Height(), Jpt.Height(), Jpt.TotalSize()); + H = 0.0; + auto mu_ad_fn = [this](std::vector &T, std::vector &W) + { + return EvalW_AD2(T,W); + }; + if (tmop_metric.Id() == 4 || tmop_metric.Id() == 14 || + tmop_metric.Id() == 66) + { + ADHessian(mu_ad_fn, H, Jpt); + this->DefaultAssembleH(H,DS,weight,A); + return; + } + MFEM_ABORT("EvalW_AD1 not implemented with this metric for " + "TMOP_WorstCaseUntangleOptimizer_Metric. " + "Please use metric 4/14/66."); +} + real_t TMOP_Metric_001::EvalW(const DenseMatrix &Jpt) const { ie.SetJacobian(Jpt.GetData()); @@ -850,6 +1019,25 @@ void TMOP_Metric_004::AssembleH(const DenseMatrix &Jpt, ie.Assemble_ddI2b(-2.0*weight, A.GetData()); } +template +type TMOP_Metric_004::EvalW_AD_impl(const std::vector &T, + const std::vector &W) const +{ + return mu4_ad(T, W); +} + +AD1Type TMOP_Metric_004::EvalW_AD1(const std::vector &T, + const std::vector &W) const +{ + return EvalW_AD_impl(T,W); +} + +AD2Type TMOP_Metric_004::EvalW_AD2(const std::vector &T, + const std::vector &W) const +{ + return EvalW_AD_impl(T,W); +} + real_t TMOP_Metric_007::EvalW(const DenseMatrix &Jpt) const { // mu_7 = |J-J^{-t}|^2 = |J|^2 + |J^{-1}|^2 - 4 @@ -971,6 +1159,25 @@ void TMOP_Metric_014::AssembleH(const DenseMatrix &Jpt, ie.Assemble_ddI1(weight, A.GetData()); } +template +type TMOP_Metric_014::EvalW_AD_impl(const std::vector &T, + const std::vector &W) const +{ + return mu14_ad(T, W); +} + +AD1Type TMOP_Metric_014::EvalW_AD1(const std::vector &T, + const std::vector &W) const +{ + return EvalW_AD_impl(T,W); +} + +AD2Type TMOP_Metric_014::EvalW_AD2(const std::vector &T, + const std::vector &W) const +{ + return EvalW_AD_impl(T,W); +} + real_t TMOP_Metric_022::EvalW(const DenseMatrix &Jpt) const { // mu_22 = (0.5*|J|^2 - det(J)) / (det(J) - tau0) @@ -1100,6 +1307,25 @@ void TMOP_Metric_055::AssembleH(const DenseMatrix &Jpt, ie.Assemble_ddI2b(2*weight*(ie.Get_I2b() - 1.0), A.GetData()); } +template +type TMOP_Metric_055::EvalW_AD_impl(const std::vector &T, + const std::vector &W) const +{ + return mu55_ad(T, W); +} + +AD1Type TMOP_Metric_055::EvalW_AD1(const std::vector &T, + const std::vector &W) const +{ + return EvalW_AD_impl(T,W); +} + +AD2Type TMOP_Metric_055::EvalW_AD2(const std::vector &T, + const std::vector &W) const +{ + return EvalW_AD_impl(T,W); +} + real_t TMOP_Metric_056::EvalWMatrixForm(const DenseMatrix &Jpt) const { // mu_56 = 0.5 (det(J) + 1 / det(J)) - 1. @@ -4096,7 +4322,7 @@ real_t TMOP_Integrator::GetElementEnergy(const FiniteElement &el, const IntegrationPoint &ip_s = ir_s->IntPoint(s); Tpr->SetIntPoint(&ip_s); - double w = surf_fit_coeff->Eval(*Tpr, ip_s) * surf_fit_normal * + real_t w = surf_fit_coeff->Eval(*Tpr, ip_s) * surf_fit_normal * 1.0 / surf_fit_dof_count[scalar_dof_id]; if (surf_fit_gf) diff --git a/fem/tmop.hpp b/fem/tmop.hpp index ae55788090..aface93645 100644 --- a/fem/tmop.hpp +++ b/fem/tmop.hpp @@ -14,10 +14,14 @@ #include "../linalg/invariants.hpp" #include "nonlininteg.hpp" +#include "../linalg/dual.hpp" namespace mfem { +using AD1Type = future::dual; +using AD2Type = future::dual; + /** @brief Abstract class for local mesh quality metrics in the target-matrix optimization paradigm (TMOP) by P. Knupp et al. */ class TMOP_QualityMetric : public HyperelasticModel @@ -69,6 +73,22 @@ public: virtual void EvalPW(const DenseMatrix &Jpt, DenseMatrix &PW) const { PW = 0.0;} + /// @brief First-derivative hook for AD-based computations. + /// @warning Not for public use. Internal use for AD-based computations. + virtual AD1Type EvalW_AD1(const std::vector &T, + const std::vector &W) const + { + MFEM_ABORT("EvalW_AD1 not implemented for this metric"); + } + + /// @brief Second-derivative hook for AD-based computations. + /// @warning Not for public use. Internal use for AD-based computations. + virtual AD2Type EvalW_AD2(const std::vector &T, + const std::vector &W) const + { + MFEM_ABORT("EvalW_AD2 not implemented for this metric"); + } + /** @brief Evaluate the derivative of the 1st Piola-Kirchhoff stress tensor and assemble its contribution to the local gradient matrix 'A'. @param[in] Jpt Represents the target->physical transformation @@ -124,6 +144,12 @@ public: void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS, const real_t weight, DenseMatrix &A) const override; + AD1Type EvalW_AD1(const std::vector &T, + const std::vector &W) const override; + + AD2Type EvalW_AD2(const std::vector &T, + const std::vector &W) const override; + /// Computes the averages of all metrics (integral of metric / volume). /// Works in parallel when called with a ParGridFunction. void ComputeAvgMetrics(const GridFunction &nodes, @@ -221,12 +247,16 @@ public: real_t EvalW(const DenseMatrix &Jpt) const override; - void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const override - { MFEM_ABORT("Not implemented"); } + AD1Type EvalW_AD1(const std::vector &T, + const std::vector &W) const override; + + AD2Type EvalW_AD2(const std::vector &T, + const std::vector &W) const override; + + void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const override; void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS, - const real_t weight, DenseMatrix &A) const override - { MFEM_ABORT("Not implemented"); } + const real_t weight, DenseMatrix &A) const override; // Compute mu_hat. real_t EvalWBarrier(const DenseMatrix &Jpt) const; @@ -368,6 +398,10 @@ class TMOP_Metric_004 : public TMOP_QualityMetric protected: mutable InvariantsEvaluator2D ie; + template + type EvalW_AD_impl(const std::vector &T, + const std::vector &W) const; + public: // W = |J|^2 - 2*det(J) real_t EvalW(const DenseMatrix &Jpt) const override; @@ -377,6 +411,12 @@ public: void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS, const real_t weight, DenseMatrix &A) const override; + AD1Type EvalW_AD1(const std::vector &T, + const std::vector &W) const override; + + AD2Type EvalW_AD2(const std::vector &T, + const std::vector &W) const override; + int Id() const override { return 4; } }; @@ -420,6 +460,10 @@ class TMOP_Metric_014 : public TMOP_QualityMetric protected: mutable InvariantsEvaluator2D ie; + template + type EvalW_AD_impl(const std::vector &T, + const std::vector &W) const; + public: // W = |J - I|^2. real_t EvalWMatrixForm(const DenseMatrix &Jpt) const override; @@ -431,6 +475,14 @@ public: void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS, const real_t weight, DenseMatrix &A) const override; + + AD1Type EvalW_AD1(const std::vector &T, + const std::vector &W) const override; + + AD2Type EvalW_AD2(const std::vector &T, + const std::vector &W) const override; + + int Id() const override { return 14; } }; /// 2D Shifted barrier form of shape metric (mu_2). @@ -479,6 +531,10 @@ class TMOP_Metric_055 : public TMOP_QualityMetric protected: mutable InvariantsEvaluator2D ie; + template + type EvalW_AD_impl(const std::vector &T, + const std::vector &W) const; + public: // W = (det(J) - 1)^2. real_t EvalW(const DenseMatrix &Jpt) const override; @@ -488,6 +544,14 @@ public: void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS, const real_t weight, DenseMatrix &A) const override; + AD1Type EvalW_AD1(const std::vector &T, + const std::vector &W) const override; + + AD2Type EvalW_AD2(const std::vector &T, + const std::vector &W) const override; + + int Id() const override { return 55; } + }; /// 2D barrier size (V) metric (polyconvex). diff --git a/fem/tmop_tools.hpp b/fem/tmop_tools.hpp index 2a554f03fa..c57dc464c5 100644 --- a/fem/tmop_tools.hpp +++ b/fem/tmop_tools.hpp @@ -111,7 +111,7 @@ public: ~InterpolatorFP() { - finder->FreeData(); + if (finder) { finder->FreeData(); } delete finder; } }; diff --git a/general/annotation.hpp b/general/annotation.hpp index 499ff9d0c1..f9b4fe8c73 100644 --- a/general/annotation.hpp +++ b/general/annotation.hpp @@ -21,9 +21,7 @@ #define MFEM_PERF_FUNCTION CALI_CXX_MARK_FUNCTION #define MFEM_PERF_BEGIN(s) CALI_MARK_BEGIN(s) #define MFEM_PERF_END(s) CALI_MARK_END(s) -#define MFEM_PERF_SCOPE(name) \ - cali::Annotation::Guard cali_autogenerated_guard_name(cali::Annotation("function").begin(std::string(name).c_str())) - +#define MFEM_PERF_SCOPE(name) CALI_CXX_MARK_SCOPE(std::string(name).c_str()) #else @@ -32,6 +30,6 @@ #define MFEM_PERF_END(s) #define MFEM_PERF_SCOPE(name) -#endif +#endif // MFEM_USE_CALIPER -#endif +#endif // MFEM_ANNOTATION_HPP diff --git a/general/array.cpp b/general/array.cpp index 10b49cb020..b8d9890602 100644 --- a/general/array.cpp +++ b/general/array.cpp @@ -170,6 +170,21 @@ int Array::IsSorted() const return 1; } +template +bool Array::IsConstant() const +{ + if (size < 2) { return true; } + const T v0 = data[0]; + for (int i = 1; i < size; i++) + { + if (data[i] != v0) + { + return false; + } + } + + return true; +} template void Array2D::Load(const char *filename, int fmt) diff --git a/general/array.hpp b/general/array.hpp index aaaea4d98d..2764d9fb2d 100644 --- a/general/array.hpp +++ b/general/array.hpp @@ -211,13 +211,16 @@ public: /// Delete the first entry with value == 'el'. inline void DeleteFirst(const T &el); + /// Delete entries at @a indices, and resize. + inline void DeleteAt(const Array &indices); + /// Delete the whole array. inline void DeleteAll(); /// Reduces the capacity of the array to exactly match the current size. inline void ShrinkToFit(); - /// Create a copy of the internal array to the provided @a copy. + /// Create a copy of the internal array to the provided @a copy. inline void Copy(Array ©) const; /// Make this Array a reference to a pointer. @@ -302,6 +305,9 @@ public: /// Does the Array have Size zero. bool IsEmpty() const { return Size() == 0; } + /// Return true if all entries of the array are the same. + bool IsConstant() const; + /// Fill the entries of the array with the cumulative sum of the entries. void PartialSum(); @@ -412,10 +418,13 @@ private: public: Array2D() { M = N = 0; } + + /// Construct an m x n 2D array. Array2D(int m, int n) : array1d(m*n) { M = m; N = n; } Array2D(const Array2D &) = default; + /// Set the 2D array size to m x n. void SetSize(int m, int n) { array1d.SetSize(m*n); M = m; N = n; } int NumRows() const { return M; } @@ -472,9 +481,11 @@ public: void Load(int new_size0,int new_size1, std::istream &in) { SetSize(new_size0,new_size1); Load(in, 1); } + /// Create a copy of the internal array to the provided @a copy. void Copy(Array2D ©) const { copy.M = M; copy.N = N; array1d.Copy(copy.array1d); } + /// Set all entries of the array to the provided constant. inline void operator=(const T &a) { array1d = a; } @@ -489,6 +500,14 @@ public: /// Prints array to stream with width elements per row void Print(std::ostream &out = mfem::out, int width = 4); + + /** @brief Find the maximal element in the array, using the comparison + operator `<` for class T. */ + T Max() const { return array1d.Max(); } + + /** @brief Find the minimal element in the array, using the comparison + operator `<` for class T. */ + T Min() const { return array1d.Min(); } }; @@ -501,15 +520,32 @@ private: public: Array3D() { N2 = N3 = 0; } + + /// Construct a 3D array of size n1 x n2 x n3. Array3D(int n1, int n2, int n3) : array1d(n1*n2*n3) { N2 = n2; N3 = n3; } + /// Set the 3D array size to n1 x n2 x n3. void SetSize(int n1, int n2, int n3) { array1d.SetSize(n1*n2*n3); N2 = n2; N3 = n3; } + /// Get the 3D array size in the first dimension. + int GetSize1() const + { + const int size = array1d.Size(); + return size == 0 ? 0 : size / (N2 * N3); + } + + /// Get the 3D array size in the second dimension. + int GetSize2() const { return N2; } + + /// Get the 3D array size in the third dimension. + int GetSize3() const { return N3; } + inline const T &operator()(int i, int j, int k) const; inline T &operator()(int i, int j, int k); + /// Set all entries of the array to the provided constant. inline void operator=(const T &a) { array1d = a; } }; @@ -935,6 +971,32 @@ inline void Array::DeleteFirst(const T &el) } } +template +inline void Array::DeleteAt(const Array &indices) +{ + HostReadWrite(); + + // Make a copy of the indices, sorted. + Array sorted_indices(indices); + sorted_indices.Sort(); + + int rm_count = 0; + for (int i = 0; i < size; i++) + { + if (rm_count < sorted_indices.Size() && i == sorted_indices[rm_count]) + { + rm_count++; + } + else + { + data[i-rm_count] = data[i]; // shift data rm_count + } + } + + // Resize to remove tail + size -= rm_count; +} + template inline void Array::DeleteAll() { diff --git a/general/communication.cpp b/general/communication.cpp index b80726aa2f..e2f3222a45 100644 --- a/general/communication.cpp +++ b/general/communication.cpp @@ -347,13 +347,23 @@ void GroupTopology::Swap(GroupTopology &other) mfem::Swap(group_mgroup, other.group_mgroup); } +/// \cond DO_NOT_DOCUMENT // Initialize the static mpi_type for the specializations of MPITypeMap: +const MPI_Datatype MPITypeMap::mpi_type = MFEM_MPI_CXX_BOOL; +const MPI_Datatype MPITypeMap::mpi_type = MPI_CHAR; +const MPI_Datatype MPITypeMap::mpi_type = MPI_UNSIGNED_CHAR; +const MPI_Datatype MPITypeMap::mpi_type = MPI_SHORT; +const MPI_Datatype MPITypeMap::mpi_type = MPI_UNSIGNED_SHORT; const MPI_Datatype MPITypeMap::mpi_type = MPI_INT; +const MPI_Datatype MPITypeMap::mpi_type = MPI_UNSIGNED; +const MPI_Datatype MPITypeMap::mpi_type = MPI_LONG; +const MPI_Datatype MPITypeMap::mpi_type = MPI_UNSIGNED_LONG; +const MPI_Datatype MPITypeMap::mpi_type = MPI_LONG_LONG; +const MPI_Datatype MPITypeMap::mpi_type = + MPI_UNSIGNED_LONG_LONG; const MPI_Datatype MPITypeMap::mpi_type = MPI_FLOAT; const MPI_Datatype MPITypeMap::mpi_type = MPI_DOUBLE; -const MPI_Datatype MPITypeMap::mpi_type = MPI_INT64_T; -const MPI_Datatype MPITypeMap::mpi_type = MPI_UINT64_T; - +/// \endcond DO_NOT_DOCUMENT GroupCommunicator::GroupCommunicator(const GroupTopology >, Mode m) : gtopo(gt), mode(m) diff --git a/general/communication.hpp b/general/communication.hpp index 9090c3ff85..58b14f55b5 100644 --- a/general/communication.hpp +++ b/general/communication.hpp @@ -607,10 +607,50 @@ protected: template struct MPITypeMap; // Specializations of MPITypeMap; mpi_type initialized in communication.cpp: +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; template<> struct MPITypeMap { static MFEM_EXPORT const MPI_Datatype mpi_type; }; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; +template<> struct MPITypeMap +{ + static MFEM_EXPORT const MPI_Datatype mpi_type; +}; template<> struct MPITypeMap { static MFEM_EXPORT const MPI_Datatype mpi_type; @@ -619,15 +659,6 @@ template<> struct MPITypeMap { static MFEM_EXPORT const MPI_Datatype mpi_type; }; -template<> struct MPITypeMap -{ - static MFEM_EXPORT const MPI_Datatype mpi_type; -}; -template<> struct MPITypeMap -{ - static MFEM_EXPORT const MPI_Datatype mpi_type; -}; - /** Reorder MPI ranks to follow the Z-curve within the physical machine topology (provided that functions to query physical node coordinates are available). diff --git a/general/forall.hpp b/general/forall.hpp index 9a5d952412..09561b7434 100644 --- a/general/forall.hpp +++ b/general/forall.hpp @@ -46,6 +46,7 @@ struct DofQuadLimits_CUDA { static constexpr int MAX_D1D = 14; static constexpr int MAX_Q1D = 14; + static constexpr int MAX_T1D = 32; static constexpr int HCURL_MAX_D1D = 5; static constexpr int HCURL_MAX_Q1D = 6; static constexpr int HDIV_MAX_D1D = 5; @@ -58,6 +59,7 @@ struct DofQuadLimits_HIP { static constexpr int MAX_D1D = 10; static constexpr int MAX_Q1D = 10; + static constexpr int MAX_T1D = 32; static constexpr int HCURL_MAX_D1D = 5; static constexpr int HCURL_MAX_Q1D = 5; static constexpr int HDIV_MAX_D1D = 5; @@ -75,6 +77,7 @@ struct DofQuadLimits_CPU static constexpr int MAX_D1D = 14; static constexpr int MAX_Q1D = 14; #endif + static constexpr int MAX_T1D = 32; static constexpr int HCURL_MAX_D1D = 10; static constexpr int HCURL_MAX_Q1D = 10; static constexpr int HDIV_MAX_D1D = 10; @@ -225,9 +228,55 @@ void OmpWrap(const int N, HBODY &&h_body) #endif } +template +void OmpWrap2D(const int Nx, const int Ny, HBODY &&h_body) +{ +#ifdef MFEM_USE_OPENMP + // requires OpenMP 3.1 + #pragma omp parallel for collapse(2) + for (int j = 0; j < Ny; j++) + { + for (int i = 0; i < Nx; i++) + { + h_body(i, j); + } + } +#else + MFEM_CONTRACT_VAR(Nx); + MFEM_CONTRACT_VAR(Ny); + MFEM_CONTRACT_VAR(h_body); + MFEM_ABORT("OpenMP requested for MFEM but OpenMP is not enabled!"); +#endif +} + +template +void OmpWrap3D(const int Nx, const int Ny, const int Nz, HBODY &&h_body) +{ +#ifdef MFEM_USE_OPENMP + // requires OpenMP 3.1 + #pragma omp parallel for collapse(3) + for (int k = 0; k < Nz; k++) + { + for (int j = 0; j < Ny; j++) + { + for (int i = 0; i < Nx; i++) + { + h_body(i, j, k); + } + } + } +#else + MFEM_CONTRACT_VAR(Nx); + MFEM_CONTRACT_VAR(Ny); + MFEM_CONTRACT_VAR(Nz); + MFEM_CONTRACT_VAR(h_body); + MFEM_ABORT("OpenMP requested for MFEM but OpenMP is not enabled!"); +#endif +} + /// RAJA Cuda and Hip backends -#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) && defined(__CUDACC__) using cuda_launch_policy = RAJA::LaunchPolicy>; using cuda_teams_x = @@ -236,7 +285,7 @@ using cuda_threads_z = RAJA::LoopPolicy; #endif -#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) && defined(__HIP__) using hip_launch_policy = RAJA::LaunchPolicy>; using hip_teams_x = @@ -245,7 +294,7 @@ using hip_threads_z = RAJA::LoopPolicy; #endif -#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) && defined(__CUDACC__) template void RajaCuWrap1D(const int N, DBODY &&d_body) { @@ -345,7 +394,7 @@ struct RajaCuWrap<3> #endif -#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) && defined(__HIP__) template void RajaHipWrap1D(const int N, DBODY &&d_body) { @@ -454,6 +503,42 @@ void RajaOmpWrap(const int N, HBODY &&h_body) RAJA::forall(RAJA::RangeSegment(0,N), h_body); } +template +void RajaOmpWrap2D(const int Nx, const int Ny, HBODY &&h_body) +{ + using omp_launch_policy = RAJA::LaunchPolicy; + using global_thread_xy = RAJA::LoopPolicy; + RAJA::RangeSegment xrange(0, Nx); + RAJA::RangeSegment yrange(0, Ny); + RAJA::launch(RAJA::ExecPlace::HOST, RAJA::LaunchParams(), + [=](RAJA::LaunchContext ctx) + { + // contiguous in x + RAJA::expt::loop(ctx, xrange, yrange, [&](int i, int j) + { + h_body(i, j); + }); + }); +} + +template +void RajaOmpWrap3D(const int Nx, const int Ny, const int Nz, HBODY &&h_body) +{ + using omp_launch_policy = RAJA::LaunchPolicy; + using global_thread_xyz = RAJA::LoopPolicy; + RAJA::RangeSegment xrange(0, Nx); + RAJA::RangeSegment yrange(0, Ny); + RAJA::RangeSegment zrange(0, Nz); + RAJA::launch(RAJA::ExecPlace::HOST, RAJA::LaunchParams(), + [=](RAJA::LaunchContext ctx) + { + // contiguous in x + RAJA::expt::loop(ctx, xrange, yrange, zrange, + [&](int i, int j, int k) + { h_body(i, j, k); }); + }); +} + #endif @@ -685,7 +770,7 @@ inline void ForallWrap(const bool use_dev, const int N, MFEM_CONTRACT_VAR(d_body); if (!use_dev) { goto backend_cpu; } -#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) && defined(__CUDACC__) // If Backend::RAJA_CUDA is allowed, use it if (Device::Allows(Backend::RAJA_CUDA)) { @@ -693,7 +778,7 @@ inline void ForallWrap(const bool use_dev, const int N, } #endif -#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) && defined(__HIP__) // If Backend::RAJA_HIP is allowed, use it if (Device::Allows(Backend::RAJA_HIP)) { @@ -753,6 +838,83 @@ inline void ForallWrap(const bool use_dev, const int N, lambda &&body, template inline void forall(int N, lambda &&body) { ForallWrap<1>(true, N, body); } +template +inline void forall(int Nx, int Ny, lambda &&body) +{ + if (Device::Allows(Backend::DEVICE_MASK)) + { + forall(Nx * Ny, [=] MFEM_HOST_DEVICE(int idx) + { + int j = idx / Nx; + int i = idx % Nx; + body(i, j); + }); + } +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP) + else if (Device::Allows(Backend::RAJA_OMP)) + { + return RajaOmpWrap2D(Nx, Ny, body); + } +#endif +#ifdef MFEM_USE_OPENMP + else if (Device::Allows(Backend::OMP)) + { + return OmpWrap2D(Nx, Ny, body); + } +#endif + else + { + for (int j = 0; j < Ny; ++j) + { + for (int i = 0; i < Nx; ++i) + { + body(i, j); + } + } + } +} + +template +inline void forall(int Nx, int Ny, int Nz, lambda &&body) +{ + if (Device::Allows(Backend::DEVICE_MASK)) + { + forall(Nx * Ny * Nz, [=] MFEM_HOST_DEVICE(int idx) + { + int i = idx % Nx; + int j = idx / Nx; + int k = j / Ny; + j = j % Ny; + body(i, j, k); + }); + } +#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP) + else if (Device::Allows(Backend::RAJA_OMP)) + { + return RajaOmpWrap3D(Nx, Ny, Nz, body); + } +#endif +#ifdef MFEM_USE_OPENMP + else if (Device::Allows(Backend::OMP)) + { + return OmpWrap3D(Nx, Ny, Nz, body); + } +#endif + else + { + for (int k = 0; k < Nz; ++k) + { + for (int j = 0; j < Ny; ++j) + { + for (int i = 0; i < Nx; ++i) + { + body(i, j, k); + } + } + } + } +} + template inline void forall_switch(bool use_dev, int N, lambda &&body) { diff --git a/general/hash.hpp b/general/hash.hpp index 13fcbeb31d..70b6af6624 100644 --- a/general/hash.hpp +++ b/general/hash.hpp @@ -57,9 +57,9 @@ struct Hashed4 * each time this class is invoked. * * There are two main methods this class provides. The Get(...) methods always - * return an item given the two or four indices. If the item didn't previously + * return an item given the two or four indices. If the item did not previously * exist, the methods creates a new one. The Find(...) methods, on the other - * hand, just return NULL or -1 if the item doesn't exist. + * hand, just return NULL or -1 if the item does not exist. * * Each new item is automatically assigned a unique ID - the index of the item * inside the BlockArray. The IDs may (but need not) be used as p1, p2, ... of @@ -95,14 +95,14 @@ public: @param[in] init_hash_size The initial size of the hash table. Must be a power of 2. */ HashTable(int block_size = 16*1024, int init_hash_size = 32*1024); - /// @brief Deep copy + /// Deep copy HashTable(const HashTable& other); - /// @brief Copy assignment not supported + /// Copy assignment not supported HashTable& operator=(const HashTable&) = delete; ~HashTable(); - /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Default - construct an item of type T if no value correspond to the requested key. + /** @brief Item accessor with key (or parents) the pair p1, p2. Default + construct an item of type T if no value corresponds to the requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -111,9 +111,9 @@ public: @warning This method should only be called if T inherits from Hashed2. */ T* Get(int p1, int p2); - /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2', - 'p3', 'p4'. The key 'p4' is optional. Default construct an item of type T - if no value corresponds to the requested key. + /** @brief Item accessor with key (or parents) the quadruplet p1, p2, p3, p4. + The key p4 is optional. Default construct an item of type T if no value + corresponds to the requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -124,10 +124,10 @@ public: @warning This method should only be called if T inherits from Hashed4. */ T* Get(int p1, int p2, int p3, int p4 = -1 /* p4 optional */); - /// Get id of item whose parents are p1, p2... Create it if it doesn't exist. - /** @brief Get the "id" of an item, this "id" corresponding to the index of the - item in the underlying BlockArray object. Default construct an item - and id if no value corresponds to the requested key. + /** @brief Get the "id" of the item whose parents are p1, p2, this "id" + corresponding to the index of the item in the underlying BlockArray + object. Default construct an item and "id" if no value corresponds to the + requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -136,9 +136,9 @@ public: @warning This method should only be called if T inherits from Hashed2. */ int GetId(int p1, int p2); - /** @brief Get the "id" of an item, this "id" corresponding to the index of the - item in the underlying BlockArray object. Default construct an item - and id if no value correspond to the requested key. + /** @brief Get the "id" of an item, this "id" corresponding to the index of + the item in the underlying BlockArray object. Default construct an item + and "id" if no value corresponds to the requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -149,9 +149,8 @@ public: @warning This method should only be called if T inherits from Hashed4. */ int GetId(int p1, int p2, int p3, int p4 = -1); - /// Find item whose parents are p1, p2... Return NULL if it doesn't exist. - /** @brief Item accessor with key (or parents) the pair 'p1', 'p2'. Return - nullptr if no value correspond to the requested key. + /** @brief Item accessor with key (or parents) the pair p1, p2. Return + NULL if no value corresponds to the requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -160,9 +159,9 @@ public: @warning This method should only be called if T inherits from Hashed2. */ T* Find(int p1, int p2); - /** @brief Item accessor with key (or parents) the quadruplet 'p1', 'p2', - 'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value - correspond to the requested key. + /** @brief Item accessor with key (or parents) the quadruplet p1, p2, p3, p4. + The key p4 is optional. Return NULL if no value corresponds to the + requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -173,8 +172,8 @@ public: @warning This method should only be called if T inherits from Hashed4. */ T* Find(int p1, int p2, int p3, int p4 = -1); - /** @brief Item const accessor with key (or parents) the pair 'p1', 'p2'. - Return nullptr if no value correspond to the requested key. + /** @brief Item const accessor with key (or parents) the pair p1, p2. + Return NULL if no value corresponds to the requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -183,9 +182,9 @@ public: @warning This method should only be called if T inherits from Hashed2. */ const T* Find(int p1, int p2) const; - /** @brief Item const accessor with key (or parents) the quadruplet 'p1', - 'p2', 'p3', 'p4'. The key 'p4' is optional. Return nullptr if no value - correspond to the requested key. + /** @brief Item const accessor with key (or parents) the quadruplet p1, p2, + p3, p4. The key p4 is optional. Return NULL if no value corresponds to the + requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -196,10 +195,12 @@ public: @warning This method should only be called if T inherits from Hashed4. */ const T* Find(int p1, int p2, int p3, int p4 = -1) const; - /// Find id of item whose parents are p1, p2... Return -1 if it doesn't exist. - /** @brief Find the "id" of an item, this "id" corresponding to the index of - the item in the underlying BlockArray object. Default construct an - item and id if no value correspond to the requested key. + /** @brief Find the "id" of an item whose parents are p1, p2. Return -1 if it + does not exist. + + This "id" corresponds to the index of the item in the underlying + BlockArray object. Default construct an item and "id" if no value + corresponds to the requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -209,8 +210,9 @@ public: int FindId(int p1, int p2) const; /** @brief Find the "id" of an item, this "id" corresponding to the index of - the item in the underlying BlockArray object. Default construct an - item and id if no value correspond to the requested key. + the item in the underlying BlockArray object. Return -1 if it does not + exist. Default construct an item and "id" if no value corresponds to the + requested key. @param[in] p1 First part of the key. @param[in] p2 Second part of the key. @@ -221,16 +223,16 @@ public: @warning This method should only be called if T inherits from Hashed4. */ int FindId(int p1, int p2, int p3, int p4 = -1) const; - /// @brief Return the number of elements currently stored in the HashTable. + /// Return the number of elements currently stored in the HashTable. int Size() const { return Base::Size() - unused.Size(); } - /// @brief Return the total number of ids (used and unused) in the HashTable. + /// Return the total number of ids (used and unused) in the HashTable. int NumIds() const { return Base::Size(); } - /// @brief Return the number of free/unused ids in the HashTable. + /// Return the number of free/unused ids in the HashTable. int NumFreeIds() const { return unused.Size(); } - /** @brief Return true if item 'id' exists in (is used by) the container. + /** @brief Return true if item @a id exists in (is used by) the container. @param[in] id Index of the item in the underlying BlockArray. @@ -241,13 +243,13 @@ public: @param[in] id Index of the item in the underlying BlockArray. - @warning Its id will be reused by newly added items. */ + @warning Its @a id will be reused by newly added items. */ void Delete(int id); - /// @brief Remove all items. + /// Remove all items. void DeleteAll(); - /** @brief Allocate an item at 'id'. Enlarge the underlying BlockArray if + /** @brief Allocate an item at @a id. Enlarge the underlying BlockArray if necessary. @param[in] id Index of the item in the underlying BlockArray. @@ -255,7 +257,7 @@ public: @param[in] p2 Second part of the key. @warning This is a special purpose method used when loading data from a - file. Does nothing if the slot 'id' has already been allocated. */ + file. Does nothing if the slot @a id has already been allocated. */ void Alloc(int id, int p1, int p2); /** @brief Reinitialize the internal list of unallocated items. @@ -287,13 +289,13 @@ public: @warning This method should only be called if T inherits from Hashed4. */ void Reparent(int id, int new_p1, int new_p2, int new_p3, int new_p4 = -1); - /// @brief Return total size of allocated memory (tables plus items), in bytes. + /// Return total size of allocated memory (tables plus items), in bytes. std::size_t MemoryUsage() const; - /// @brief Write details of the memory usage to the mfem output stream. + /// Write details of the memory usage to the mfem output stream. void PrintMemoryDetail() const; - /// @brief Print a histogram of bin sizes for debugging purposes. + /// Print a histogram of bin sizes for debugging purposes. void PrintStats() const; class iterator : public Base::iterator @@ -346,7 +348,7 @@ public: protected: /** The hash table: each bin is a linked list of items. For each non-empty - bin, this arrays stores the 'id' of the first item in the list, or -1 + bin, this arrays stores the "id" of the first item in the list, or -1 if the bin is empty. */ int* table; @@ -384,11 +386,11 @@ protected: { return (984120265ul*p1 + 125965121ul*p2 + 495698413ul*p3) & mask; } // Delete() and Reparent() use one of these: - /// @brief Hash function for items of type T that inherit from Hashed2. + /// Hash function for items of type T that inherit from Hashed2. inline int Hash(const Hashed2& item) const { return Hash(item.p1, item.p2); } - /// @brief Hash function for items of type T that inherit from Hashed4. + /// Hash function for items of type T that inherit from Hashed4. inline int Hash(const Hashed4& item) const { return Hash(item.p1, item.p2, item.p3); } @@ -415,15 +417,15 @@ protected: @warning This method should only be called if T inherits from Hashed4. */ int SearchList(int id, int p1, int p2, int p3) const; - /** @brief Insert the item 'id' into bin 'idx'. + /** @brief Insert the item @a id into bin @a idx. @param[in] idx The bin/bucket index. @param[in] id The index of the item in the BlockArray. @param[in] item The item to insert at the beginning of the linked list. - @warning The method only works with bin 'idx' and does not check the - overall fill factor of the hash table. If appropriate, - use CheckRehash() for that. */ + @warning The method only works with bin @a idx and does not check the + overall fill factor of the hash table. If appropriate, use + CheckRehash() for that. */ inline void Insert(int idx, int id, T &item); /** @brief Unlink an item @a id from the linked list of bin @a idx. @@ -444,11 +446,11 @@ protected: and reinsert all items into the new bins. NOTE: Rehashing is computationally expensive (O(N) in the number of items), - but since it is only done rarely (when the number of items doubles), - the amortized complexity of inserting an item is still O(1). */ + but since it is only done rarely (when the number of items doubles), the + amortized complexity of inserting an item is still O(1). */ void DoRehash(); - /** @brief Return the size of the bin "idx". + /** @brief Return the size of the bin @a idx. @param[in] idx The index of the bin. @return The size of the bin. */ diff --git a/general/mem_manager.cpp b/general/mem_manager.cpp index 05ef691ace..7e828a6886 100644 --- a/general/mem_manager.cpp +++ b/general/mem_manager.cpp @@ -513,10 +513,7 @@ public: void *HtoD(void *dst, const void *src, size_t bytes) override { return HipMemcpyHtoD(dst, src, bytes); } void *DtoD(void* dst, const void* src, size_t bytes) override - // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so - // instead we use hipMemcpyAsync to get similar behavior. - // for more info see: https://github.com/mfem/mfem/pull/2780 - { return HipMemcpyDtoDAsync(dst, src, bytes); } + { return HipMemcpyDtoD(dst, src, bytes); } void *DtoH(void *dst, const void *src, size_t bytes) override { return HipMemcpyDtoH(dst, src, bytes); } }; @@ -658,10 +655,7 @@ public: return CuMemcpyDtoD(dst, src, bytes); #endif #ifdef MFEM_USE_HIP - // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so - // instead we use hipMemcpyAsync to get similar behavior. - // for more info see: https://github.com/mfem/mfem/pull/2780 - return HipMemcpyDtoDAsync(dst, src, bytes); + return HipMemcpyDtoD(dst, src, bytes); #endif // rm.copy(dst, const_cast(src), bytes); return dst; } @@ -1390,8 +1384,11 @@ void MemoryManager::Insert(void *h_ptr, size_t bytes, { auto &m = res.first->second; MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt && - (m.d_mt == d_mt || (d_mt == MemoryType::DEFAULT && - m.d_mt == GetDualMemoryType(h_mt))), + (m.d_mt == d_mt || + (d_mt == MemoryType::DEFAULT && + m.d_mt == GetDualMemoryType(h_mt)) || + (m.d_mt == MemoryType::DEFAULT && + d_mt == GetDualMemoryType(m.h_mt))), "Address already present with different attributes!"); #ifdef MFEM_TRACK_MEM_MANAGER mfem::out << "[mfem memory manager]: repeated registration of h_ptr: " diff --git a/general/reducers.hpp b/general/reducers.hpp index 50f357fa02..9b56596c6c 100644 --- a/general/reducers.hpp +++ b/general/reducers.hpp @@ -466,7 +466,7 @@ template struct reduction_kernel /// helper for computing the reduction block size static int block_log2(unsigned N) { -#if defined(__GNUC__) or defined(__clang__) +#if defined(__GNUC__) || defined(__clang__) return N ? (sizeof(unsigned) * 8 - __builtin_clz(N)) : 0; #elif defined(_MSC_VER) return sizeof(unsigned) * 8 - __lzclz(N); diff --git a/general/scan.hpp b/general/scan.hpp new file mode 100644 index 0000000000..d24590dadc --- /dev/null +++ b/general/scan.hpp @@ -0,0 +1,409 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_SCAN_HPP +#define MFEM_SCAN_HPP + +#include "backends.hpp" +#include "forall.hpp" + +#ifdef MFEM_USE_CUDA +#include +#include +#define MFEM_CUB_NAMESPACE cub +#elif defined(MFEM_USE_HIP) +#include +#include +#define MFEM_CUB_NAMESPACE hipcub +#endif + +#include +#include +#include +#include + +namespace mfem +{ +/// Equivalent to InclusiveScan(use_dev, d_in, d_out, num_items, workspace, +/// std::plus<>{}) +template +void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items) +{ + // forward to InclusiveSum for potentially faster kernels +#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) + if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) + { + static Array workspace; + size_t bytes = workspace.Size(); + if (bytes) + { + auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum( + workspace.Write(), bytes, d_in, d_out, num_items); +#if defined(MFEM_USE_CUDA) + if (err == cudaSuccess) + { + return; + } +#elif defined(MFEM_USE_HIP) + if (err == hipSuccess) + { + return; + } +#endif + } + // try allocating a larger buffer + bytes = 0; + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum( + nullptr, bytes, d_in, d_out, num_items)); + workspace.SetSize(bytes); + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveSum( + workspace.Write(), bytes, d_in, d_out, num_items)); + return; + } +#endif +#if 0 + std::inclusive_scan(d_in, d_in + num_items, d_out); +#else + // work-around to some compilers not fully supporting C++17 + if (num_items) + { + *d_out = *d_in; + auto prev = d_out; + ++d_in; + ++d_out; + for (size_t i = 1; i < num_items; ++i) + { + *d_out = (*prev) + (*d_in); + prev = d_out; + ++d_in; + ++d_out; + } + } +#endif +} + +/// @brief Performs an inclusive scan of [d_in, d_in+num_items) -> [d_out, +/// d_out+num_items). This call is potentially asynchronous on the device. +/// +/// @a d_in input start. +/// @a d_out output start. Can perform in-place scans with d_out = d_in +/// @a scan_op binary scan functor. Must be associative. If only weakly +/// associative (i.e. floating point addition) results are not deterministic. On +/// device this must also be commutative. +template +void InclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items, + ScanOp scan_op) +{ +#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) + if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) + { + static Array workspace; + size_t bytes = workspace.Size(); + if (bytes) + { + auto err = MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan( + workspace.Write(), bytes, d_in, d_out, scan_op, num_items); +#if defined(MFEM_USE_CUDA) + if (err == cudaSuccess) + { + return; + } +#elif defined(MFEM_USE_HIP) + if (err == hipSuccess) + { + return; + } +#endif + } + // try allocating a larger buffer + bytes = 0; + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan( + nullptr, bytes, d_in, d_out, scan_op, num_items)); + workspace.SetSize(bytes); + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::InclusiveScan( + workspace.Write(), bytes, d_in, d_out, scan_op, num_items)); + return; + } +#endif +#if 0 + std::inclusive_scan(d_in, d_in + num_items, d_out, scan_op); +#else + // work-around to some compilers not fully supporting C++17 + if (num_items) + { + *d_out = *d_in; + auto prev = d_out; + ++d_in; + ++d_out; + for (size_t i = 1; i < num_items; ++i) + { + *d_out = scan_op(*prev, *d_in); + prev = d_out; + ++d_in; + ++d_out; + } + } +#endif +} + +/// Performs an exclusive scan of [d_in, d_in+num_items) -> [d_out, +/// d_out+num_items). This call is potentially asynchronous on the device. +/// @a d_in input start. +/// @a d_out output start. Can perform in-place scans with d_out = d_in +/// @a scan_op binary scan functor. Must be associative. If only weakly +/// associative (i.e. floating point addition) results are not deterministic. On +/// device this must also be commutative. +template +void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items, + T init_value, ScanOp scan_op) +{ +#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) + if (use_dev && mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) + { + static Array workspace; + size_t bytes = workspace.Size(); + if (bytes) + { + auto err = MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan( + workspace.Write(), bytes, d_in, d_out, scan_op, init_value, + num_items); +#if defined(MFEM_USE_CUDA) + if (err == cudaSuccess) + { + return; + } +#elif defined(MFEM_USE_HIP) + if (err == hipSuccess) + { + return; + } +#endif + } + // try allocating a larger buffer + bytes = 0; + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan( + nullptr, bytes, d_in, d_out, scan_op, init_value, num_items)); + workspace.SetSize(bytes); + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceScan::ExclusiveScan( + workspace.Write(), bytes, d_in, d_out, scan_op, init_value, + num_items)); + return; + } +#endif +#if 0 + std::exclusive_scan(d_in, d_in + num_items, d_out, init_value, scan_op); +#else + // work-around to some compilers not fully supporting C++17 + if (num_items) + { + for (size_t i = 0; i < num_items; ++i) + { + auto next = scan_op(init_value, *d_in); + *d_out = init_value; + init_value = next; + ++d_out; + ++d_in; + } + } +#endif +} + +/// Equivalent to ExclusiveScan(use_dev, d_in, d_out, num_items, init_value, +/// workspace, std::plus<>{}) +template +void ExclusiveScan(bool use_dev, InputIt d_in, OutputIt d_out, size_t num_items, + T init_value) +{ + ExclusiveScan(use_dev, d_in, d_out, num_items, init_value, std::plus<> {}); +} + +/// @brief Equivalent to *d_num_selected_out = std::copy_if(d_in, +/// d_in+num_items, d_out, [=](auto iter){ return d_flags[iter-d_in]; }) - +/// d_out; +/// +/// None of the following ranges may overlap: +/// - [d_in, d_in+num_items) +/// - [d_flags, d_flags+num_items) +/// - [d_out, d_out+*d_num_selected_out) +/// - [d_num_selected_out, d_num_selected_out+1) +template +void CopyFlagged(bool use_dev, InputIt d_in, FlagIt d_flags, OutputIt d_out, + NumSelectedIt d_num_selected_out, size_t num_items) +{ +#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) + if (use_dev && + mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) + { + static Array workspace; + size_t bytes = workspace.Size(); + if (bytes) + { + auto err = MFEM_CUB_NAMESPACE::DeviceSelect::Flagged( + workspace.Write(), bytes, d_in, d_flags, d_out, d_num_selected_out, + num_items); +#if defined(MFEM_USE_CUDA) + if (err == cudaSuccess) + { + return; + } +#elif defined(MFEM_USE_HIP) + if (err == hipSuccess) + { + return; + } +#endif + } + // try allocating a larger buffer + bytes = 0; + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceSelect::Flagged( + nullptr, bytes, d_in, d_flags, d_out, d_num_selected_out, num_items)); + workspace.SetSize(bytes); + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceSelect::Flagged( + workspace.Write(), bytes, d_in, d_flags, d_out, d_num_selected_out, + num_items)); + return; + } +#endif + *d_num_selected_out = 0; + for (size_t i = 0; i < num_items; ++i, ++d_in, ++d_flags) + { + if (*d_flags) + { + *d_out = *d_in; + ++d_out; + ++*d_num_selected_out; + } + } +} + +/// @brief Equivalent to *d_num_selected_out = std::copy_if(d_in, +/// d_in+num_items, d_out, select_op) - d_out; +/// +/// None of the following ranges may overlap: +/// - [d_in, d_in+num_items) +/// - [d_out, d_out+*d_num_selected_out) +/// - [d_num_selected_out, d_num_selected_out+1) +template +void CopyIf(bool use_dev, InputIt d_in, OutputIt d_out, + NumSelectedIt d_num_selected_out, size_t num_items, + SelectOp select_op) +{ +#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) + if (use_dev && + mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) + { +#if defined(MFEM_USE_CUDA) && \ + (__CUDACC_VER_MAJOR__ < 12 || \ + (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ < 5)) + // bug in cuda < 12.5, work-around: use Flagged instead + Array flags(num_items); + auto ptr = flags.Write(); + forall(num_items, + [=] MFEM_HOST_DEVICE(int i) { ptr[i] = select_op(d_in[i]); }); + CopyFlagged(use_dev, d_in, ptr, d_out, d_num_selected_out, num_items); +#else + static Array workspace; + size_t bytes = workspace.Size(); + if (bytes) + { + auto err = MFEM_CUB_NAMESPACE::DeviceSelect::If( + workspace.Write(), bytes, d_in, d_out, d_num_selected_out, + num_items, select_op); +#if defined(MFEM_USE_CUDA) + if (err == cudaSuccess) + { + return; + } +#elif defined(MFEM_USE_HIP) + if (err == hipSuccess) + { + return; + } +#endif + } + // try allocating a larger buffer + bytes = 0; + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceSelect::If( + nullptr, bytes, d_in, d_out, d_num_selected_out, num_items, + select_op)); + workspace.SetSize(bytes); + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceSelect::If( + workspace.Write(), bytes, d_in, d_out, d_num_selected_out, num_items, + select_op)); +#endif + return; + } +#endif + *d_num_selected_out = 0; + for (size_t i = 0; i < num_items; ++i, ++d_in) + { + if (select_op(*d_in)) + { + *d_out = *d_in; + ++d_out; + ++*d_num_selected_out; + } + } +} + +/// @brief equivalent to *d_num_selected_out = std::unique_copy(d_in, +/// d_in+num_items, d_out) - d_out; +/// +/// None of the following ranges may overlap: +/// - [d_in, d_in+num_items) +/// - [d_out, d_out+*d_num_selected_out) +/// - [d_num_selected_out, d_num_selected_out+1) +template +void CopyUnique(bool use_dev, InputIt d_in, OutputIt d_out, + NumSelectedIt d_num_selected_out, size_t num_items) +{ +#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) + if (use_dev && + mfem::Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) + { + static Array workspace; + size_t bytes = workspace.Size(); + if (bytes) + { + auto err = MFEM_CUB_NAMESPACE::DeviceSelect::Unique( + workspace.Write(), bytes, d_in, d_out, d_num_selected_out, + num_items); +#if defined(MFEM_USE_CUDA) + if (err == cudaSuccess) + { + return; + } +#elif defined(MFEM_USE_HIP) + if (err == hipSuccess) + { + return; + } +#endif + } + // try allocating a larger buffer + bytes = 0; + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceSelect::Unique( + nullptr, bytes, d_in, d_out, d_num_selected_out, num_items)); + workspace.SetSize(bytes); + MFEM_GPU_CHECK(MFEM_CUB_NAMESPACE::DeviceSelect::Unique( + workspace.Write(), bytes, d_in, d_out, d_num_selected_out, + num_items)); + return; + } +#endif + *d_num_selected_out = + std::unique_copy(d_in, d_in + num_items, d_out) - d_out; +} +} // namespace mfem + +#undef MFEM_CUB_NAMESPACE + +#endif diff --git a/linalg/batched/batched.hpp b/linalg/batched/batched.hpp index c5f9fcd125..f31a1e0114 100644 --- a/linalg/batched/batched.hpp +++ b/linalg/batched/batched.hpp @@ -87,7 +87,7 @@ public: /// pivots are stored in @a P. /// /// $A$ is represented by the DenseTensor @a A with shape (n, n, n_mat). On - /// output, $P$ has shape (n, n_mat). + /// output, $P$ has shape (n, n_mat). Pivots always use 1-based indexing. static void LUFactor(DenseTensor &A, Array &P); /// @brief Replaces $x$ with $A^{-1} x$, given the LU factors @a A and pivots /// @a P of the block-diagonal matrix $A$. @@ -96,9 +96,8 @@ public: /// LUFactor(). $A$ has shape (n, n, n_mat) and $x$ has shape (n, n_rhs, /// n_mat). /// - /// @warning LUSolve() and LUFactor() should be called using the same backend - /// because of potential incompatibilities (e.g. 0-based or 1-based - /// indexing). + /// @warning P should use 1-based indexing. This is what LUFactor() generates + /// for all available backends. static void LUSolve(const DenseTensor &A, const Array &P, Vector &x); /// @brief Returns true if the requested backend is available. /// diff --git a/linalg/batched/magma.cpp b/linalg/batched/magma.cpp index 57ba8b0eea..e87c404a61 100644 --- a/linalg/batched/magma.cpp +++ b/linalg/batched/magma.cpp @@ -68,7 +68,7 @@ void MagmaBatchedLinAlg::AddMult(const DenseTensor &A, const Vector &x, auto d_x = x.Read(); // Shape (n, k, n_mat); auto d_y = beta == 0.0 ? y.Write() : y.ReadWrite(); // Shape (m, k, n_mat); - magma_trans_t magma_op = tr ? MagmaNoTrans : MagmaTrans; + magma_trans_t magma_op = tr ? MagmaTrans : MagmaNoTrans; MFEM_MAGMABLAS_PREFIX(gemm_batched_strided)( magma_op, MagmaNoTrans, m, k, n, alpha, d_A, m, m*n, d_x, n, n*k, @@ -167,7 +167,7 @@ void MagmaBatchedLinAlg::Invert(DenseTensor &A) const magma_int_t status; status = MFEM_MAGMA_PREFIX(getrf_batched)( - n, n, d_A_ptrs, n, d_P_ptrs, info_array.Write(), n_mat, + n, n, d_LU_ptrs, n, d_P_ptrs, info_array.Write(), n_mat, Magma::Queue()); MFEM_VERIFY(status == MAGMA_SUCCESS, ""); diff --git a/linalg/batched/native.cpp b/linalg/batched/native.cpp index 2b02ee8bd5..833e7ee0db 100644 --- a/linalg/batched/native.cpp +++ b/linalg/batched/native.cpp @@ -132,7 +132,7 @@ void NativeBatchedLinAlg::Invert(DenseTensor &A) const // X <- X P for (int k = m - 1; k >= 0; k--) { - const int piv_k = ipiv[k]; + const int piv_k = ipiv[k] - 1; if (k != piv_k) { for (int i = 0; i < m; i++) diff --git a/linalg/complex_densemat.cpp b/linalg/complex_densemat.cpp index 5f1c6e2d60..a534241efd 100644 --- a/linalg/complex_densemat.cpp +++ b/linalg/complex_densemat.cpp @@ -444,7 +444,7 @@ bool ComplexLUFactors::Factor(int m, real_t TOL) piv = j; } } - ipiv[i] = piv; + ipiv[i] = piv + 1; if (piv != i) { // swap rows i and piv in both L and U parts diff --git a/linalg/complex_densemat.hpp b/linalg/complex_densemat.hpp index 29580d5217..22aa3fb9ad 100644 --- a/linalg/complex_densemat.hpp +++ b/linalg/complex_densemat.hpp @@ -124,11 +124,7 @@ class ComplexLUFactors : public ComplexFactors { public: int *ipiv; -#ifdef MFEM_USE_LAPACK - static const int ipiv_base = 1; -#else - static const int ipiv_base = 0; -#endif + static constexpr int ipiv_base = 1; /** With this constructor, the (public) data and ipiv members should be set explicitly before calling class methods. */ diff --git a/linalg/densemat.cpp b/linalg/densemat.cpp index b1092cc4a8..aed13eba6d 100644 --- a/linalg/densemat.cpp +++ b/linalg/densemat.cpp @@ -3406,7 +3406,7 @@ bool LUFactors::Factor(int m, real_t TOL) piv = j; } } - ipiv[i] = piv; + ipiv[i] = piv + 1; if (piv != i) { // swap rows i and piv in both L and U parts @@ -3446,7 +3446,7 @@ real_t LUFactors::Det(int m) const real_t det = 1.0; for (int i=0; i(GetData(k), SizeI()*SizeJ(), false); - return Mk; + return operator()(k, Mk); } const DenseMatrix &operator()(int k) const + { + return operator()(k, Mk); + } + DenseMatrix &operator()(int k, DenseMatrix& buff) { MFEM_ASSERT_INDEX_IN_RANGE(k, 0, SizeK()); - Mk.data = Memory(const_cast(GetData(k)), SizeI()*SizeJ(), - false); - return Mk; + buff.UseExternalData(nullptr, SizeI(), SizeJ()); + buff.data = Memory(GetData(k), SizeI()*SizeJ(), false); + return buff; + } + const DenseMatrix &operator()(int k, DenseMatrix& buff) const + { + MFEM_ASSERT_INDEX_IN_RANGE(k, 0, SizeK()); + buff.UseExternalData(nullptr, SizeI(), SizeJ()); + buff.data = Memory(const_cast(GetData(k)), SizeI()*SizeJ(), + false); + return buff; } real_t &operator()(int i, int j, int k) diff --git a/linalg/hypre.cpp b/linalg/hypre.cpp index 35472cbe86..05c878a305 100644 --- a/linalg/hypre.cpp +++ b/linalg/hypre.cpp @@ -561,7 +561,8 @@ void CopyMemory(Memory &src, Memory &dst, MemoryClass dst_mc, this function. In particular, @a dst should be empty or deleted before calling this function. */ template -void CopyConvertMemory(Memory &src, MemoryClass dst_mc, Memory &dst) +void CopyConvertMemory(const Memory &src, MemoryClass dst_mc, + Memory &dst) { auto capacity = src.Capacity(); dst.New(capacity, GetMemoryType(dst_mc)); @@ -842,8 +843,8 @@ static int GetPartitioningArraySize(MPI_Comm comm) /// /// Both @a row and @a col are partitioning arrays, whose length is returned by /// GetPartitioningArraySize(), see @ref hypre_partitioning_descr. -static bool RowAndColStartsAreEqual(MPI_Comm comm, HYPRE_BigInt *rows, - HYPRE_BigInt *cols) +static bool RowAndColStartsAreEqual(MPI_Comm comm, const HYPRE_BigInt *rows, + const HYPRE_BigInt *cols) { const int part_size = GetPartitioningArraySize(comm); bool are_equal = true; @@ -1131,7 +1132,7 @@ HypreParMatrix::HypreParMatrix( HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_BigInt *row_starts, HYPRE_BigInt *col_starts, - SparseMatrix *sm_a) + const SparseMatrix *sm_a) { MFEM_ASSERT(sm_a != NULL, "invalid input"); MFEM_VERIFY(!HYPRE_AssumedPartitionCheck(), @@ -1145,7 +1146,7 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, hypre_CSRMatrixSetDataOwner(csr_a,0); MemoryIJData mem_a; - CopyCSR(sm_a, mem_a, csr_a, false); + CopyCSR(const_cast(sm_a), mem_a, csr_a, false); hypre_CSRMatrixSetRownnz(csr_a); // NOTE: this call creates a matrix on host even when device support is @@ -1307,10 +1308,11 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np, HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows, HYPRE_BigInt glob_nrows, HYPRE_BigInt glob_ncols, - int *I, HYPRE_BigInt *J, - real_t *data, - HYPRE_BigInt *rows, - HYPRE_BigInt *cols) + const int *I, + const HYPRE_BigInt *J, + const real_t *data, + const HYPRE_BigInt *rows, + const HYPRE_BigInt *cols) { Init(); @@ -2327,8 +2329,8 @@ void HypreParMatrix::Threshold(real_t threshold) /* TODO: GenerateDiagAndOffd() uses an int array of size equal to the number of columns in csr_A_wo_z which is the global number of columns in A. This does not scale well. */ - ierr += GenerateDiagAndOffd(csr_A_wo_z,parcsr_A_ptr, - col_start,col_end); + ierr += hypre_GenerateDiagAndOffd(csr_A_wo_z,parcsr_A_ptr, + col_start,col_end); ierr += hypre_CSRMatrixDestroy(csr_A_wo_z); diff --git a/linalg/hypre.hpp b/linalg/hypre.hpp index 020dc64808..a7c334d3c6 100644 --- a/linalg/hypre.hpp +++ b/linalg/hypre.hpp @@ -25,11 +25,18 @@ #define HYPRE_TIMING // hypre header files +#if MFEM_HYPRE_VERSION < 30000 #include #include +#else +#include <_hypre_seq_mv.h> +#include <_hypre_lobpcg_temp_multivector.h> +#endif #include <_hypre_parcsr_mv.h> #include <_hypre_parcsr_ls.h> +#include + #ifdef HYPRE_COMPLEX #error "MFEM does not work with HYPRE's complex numbers support" #endif @@ -53,6 +60,10 @@ #error "MFEM_USE_HIP=YES is required when HYPRE is built with HIP!" #endif +#if MFEM_HYPRE_VERSION > 21500 +#define HYPRE_AssumedPartitionCheck() 1 +#endif + namespace mfem { @@ -554,7 +565,7 @@ public: partitioning arrays @a row_starts and @a col_starts. */ HypreParMatrix(MPI_Comm comm, HYPRE_BigInt *row_starts, HYPRE_BigInt *col_starts, - SparseMatrix *a); // constructor with 4 arguments, v2 + const SparseMatrix *a); // constructor with 4 arguments, v2 /// Creates boolean block-diagonal rectangular parallel matrix. /** The new HypreParMatrix does not take ownership of any of the input @@ -583,9 +594,9 @@ public: arrays (so they can be deleted). See @ref hypre_partitioning_descr "here" for a description of the partitioning arrays @a rows and @a cols. */ HypreParMatrix(MPI_Comm comm, int nrows, HYPRE_BigInt glob_nrows, - HYPRE_BigInt glob_ncols, int *I, HYPRE_BigInt *J, - real_t *data, HYPRE_BigInt *rows, - HYPRE_BigInt *cols); // constructor with 9 arguments + HYPRE_BigInt glob_ncols, const int *I, const HYPRE_BigInt *J, + const real_t *data, const HYPRE_BigInt *rows, + const HYPRE_BigInt *cols); // constructor with 9 arguments /** @brief Copy constructor for a ParCSR matrix which creates a deep copy of structure and data from @a P. */ diff --git a/linalg/hypre_parcsr.cpp b/linalg/hypre_parcsr.cpp index a11eab8bf6..2409e25456 100644 --- a/linalg/hypre_parcsr.cpp +++ b/linalg/hypre_parcsr.cpp @@ -1916,9 +1916,9 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A, /* FIXME: GenerateDiagAndOffd() uses an int array of size equal to the number of columns in csr_C_temp which is the global number of columns in A and B. This does not scale well. */ - ierr += GenerateDiagAndOffd(csr_C_temp, C, - hypre_ParCSRMatrixFirstColDiag(A), - hypre_ParCSRMatrixLastColDiag(A)); + ierr += hypre_GenerateDiagAndOffd(csr_C_temp, C, + hypre_ParCSRMatrixFirstColDiag(A), + hypre_ParCSRMatrixLastColDiag(A)); /* delete CSR version of C */ ierr += hypre_CSRMatrixDestroy(csr_C_temp); diff --git a/linalg/hypre_parcsr.hpp b/linalg/hypre_parcsr.hpp index 4dadd31f71..07b321704f 100644 --- a/linalg/hypre_parcsr.hpp +++ b/linalg/hypre_parcsr.hpp @@ -21,6 +21,10 @@ // hypre header files #include <_hypre_parcsr_mv.h> +#if MFEM_HYPRE_VERSION < 30000 +#define hypre_GenerateDiagAndOffd GenerateDiagAndOffd +#endif + // Older hypre versions do not define HYPRE_BigInt and HYPRE_MPI_BIG_INT, so we // define them here for backward compatibility. #if MFEM_HYPRE_VERSION < 21600 diff --git a/linalg/kernels.hpp b/linalg/kernels.hpp index 415dff999b..14982faf37 100644 --- a/linalg/kernels.hpp +++ b/linalg/kernels.hpp @@ -1765,7 +1765,7 @@ inline void LSolve(const real_t *data, const int m, const int *ipiv, real_t *x) // X <- P X for (int i = 0; i < m; i++) { - internal::Swap(x[i], x[ipiv[i]]); + internal::Swap(x[i], x[ipiv[i] - 1]); } // X <- L^{-1} X for (int j = 0; j < m; j++) @@ -1904,7 +1904,7 @@ inline bool LUFactor(real_t *A, const int m, int *ipiv, const real_t tol=0.0) piv = j; } } - ipiv[i] = piv; + ipiv[i] = piv + 1; if (piv != i) { // swap rows i and piv in both L and U parts diff --git a/linalg/mma.cpp b/linalg/mma.cpp index 1f6ab81f3c..63351e94f8 100644 --- a/linalg/mma.cpp +++ b/linalg/mma.cpp @@ -1019,7 +1019,7 @@ MMA::MMA(MPI_Comm comm_, int nVar, int nCon, real_t *xval, int iter) mSubProblem.reset(new MMA::MMASubSvanberg(*this, nVar, nCon)); } -MMA::MMA(MPI_Comm comm_, const int & nVar, const int & nCon, +MMA::MMA(MPI_Comm comm_, const int nVar, const int nCon, const Vector & xval, int iter) : MMA(comm_, nVar, nCon, xval.GetData(), iter) {} #endif diff --git a/linalg/mma.hpp b/linalg/mma.hpp index a6ba06213c..d9e4a9ed7e 100644 --- a/linalg/mma.hpp +++ b/linalg/mma.hpp @@ -25,25 +25,42 @@ namespace mfem // forward declaration class Vector; -/** \brief MMA (Method of Moving Asymptotes) solves an optimization problem - * of the form: +/** \brief MMA (Method of Moving Asymptotes) solves a nonlinear optimization + * problem involving an objective function, inequality constraints, + * and variable bounds. * - * Find x that minimizes the objective function F(x), - * subject to C(x)_i <= 0, for all i = 1, ... m - * x_lo <= x <= x_hi. + * \details + * This class finds ${\bf x} \in R^n$ that solves the following nonlinear + * program: + * $$ + * \begin{array}{ll} + * \min_{{\bf x} \in R^n} & F({\bf x})\\ + * \textrm{subject to} & C({\bf x})_i \leq 0,\quad + * \textrm{for all}\quad i = 1,\ldots m\\ + * & {\bf x}_{\textrm{lo}} \leq {\bf x} \leq + * {\bf x}_{\textrm{hi}}. + * \end{array} + * $$ + * Here $F : R^n \to R$ is the objective function, and + * $C : R^n \to R^m$ is a set of $m$ inequality constraints. The + * variable bounds are sometimes called box constraints. By + * convention, the routine seeks ${\bf x}$ that minimizes the + * objective function, $F$. Maximization problems should be + * reformulated as a minimization of $-F$. * * The objective functions are replaced by convex functions * chosen based on gradient information, and solved using a dual method. * The unique optimal solution of this subproblem is returned as the next * iteration point. Optimality is determined by the KKT conditions. * - * The "Update" function in MMA advances the optimization and must be called - * in every optimization iteration. Current and previous iteration points - * construct the "moving asymptotes". The design variables, objective function, - * constraints are passed to an approximating subproblem. The design variables - * are updated and returned. Its implementation closely follows the original - * formulation of 'Svanberg, K. (2007). MMA and GCMMA-two methods - * for nonlinear optimization. vol, 1, 1-15.' + * The "Update" function in MMA advances the optimization and must be + * called in every optimization iteration. Current and previous iteration + * points construct the "moving asymptotes". The design variables, + * objective function, constraints are passed to an approximating + * subproblem. The design variables are updated and returned. Its + * implementation closely follows the original formulation of 'Svanberg, K. (2007). + * MMA and GCMMA-two methods for nonlinear optimization. vol, 1, 1-15.' * * When used in parallel, all Vectors are assumed to be true dof vectors, * and the operators are expected to be defined for tdof vectors. @@ -52,46 +69,164 @@ class Vector; class MMA { public: - /// Serial constructor: - /// nVar - number of design parameters; - /// nCon - number of constraints; - /// xval[nVar] - initial parameter values + /** + * \brief Serial constructor + * \param nVar total number of design parameters + * \param nCon number of inequality constraints (i.e., $C$) + * \param xval initial values for design parameters (a pointer + * to \p nVar doubles). Caller retains ownership of + * this pointer/data. + * \param iterationNumber the starting iteration number + */ MMA(int nVar, int nCon, real_t *xval, int iterationNumber = 0); + + /** + * \brief Serial constructor + * \param nVar total number of design parameters + * \param nCon number of inequality constraints (i.e., $C$) + * \param xval initial values for design parameters (size should + * be \p nVar). Caller retains ownership of + * this Vector. + * \param iterationNumber the starting iteration number + */ MMA(const int nVar, int nCon, Vector & xval, int iterationNumber = 0); #ifdef MFEM_USE_MPI - /// Parallel constructor: - /// comm_ - communicator + /** + * \brief Parallel constructor + * \param comm_ the MPI communicator participating in the NLP solve + * \param nVar number of design parameters on this MPI rank + * \param nCon total number of inequality constraints (i.e., $C$). + * Every MPI rank provides the same value here. + * \param xval initial values for design parameters on this MPI rank + * (a pointer to \p nVar doubles). Caller retains ownership + * of this pointer/data. + * \param iterationNumber the starting iteration number. All MPI ranks + * should pass in the same value here. + * + * \details + * Each MPI rank has a subset of the total design variable vector, and + * calls for that MPI rank always address its subset of the design + * variable vector and gradients with respect to its subset of the design + * variable vector. + * + * If you wanted to determine the global number of design variables, it + * would be determined as follows: + * \code{.cpp} + * int globalDesignVars; + * MPI_Allreduce(&nVar, &globalDesignVars, 1, MPI_INT, MPI_SUM, comm_); + * \endcode + */ MMA(MPI_Comm comm_, int nVar, int nCon, real_t *xval, int iterationNumber = 0); - MMA(MPI_Comm comm_, const int & nVar, const int & nCon, const Vector & xval, + /** + * \brief Parallel constructor + * \param comm_ the MPI communicator participating in the NLP solve + * \param nVar number of design parameters on this MPI rank + * \param nCon total number of inequality constraints (i.e., $C$). + * Every MPI rank provides the same value here. + * \param xval initial values for design parameters (size should + * be \p nVar). Caller retains ownership of + * this Vector. + * \param iterationNumber the starting iteration number. All MPI ranks + * should pass in the same value here. + * + * \details + * Each MPI rank has a subset of the total design variable vector, and + * calls for that MPI rank always address its subset of the design + * variable vector and gradients with respect to its subset of the design + * variable vector. + * + * If you wanted to determine the global number of design variables, it + * would be determined as follows: + * \code{.cpp} + * int globalDesignVars; + * MPI_Allreduce(&nVar, &globalDesignVars, 1, MPI_INT, MPI_SUM, comm_); + * \endcode + */ + MMA(MPI_Comm comm_, const int nVar, const int nCon, const Vector & xval, int iterationNumber = 0); #endif /// Destructor ~MMA(); - /// Update the optimization parameters - /// dfdx[nVar] - gradients of the objective - /// gx[nCon] - values of the constraints - /// dgdx[nCon*nVar] - gradients of the constraints ordered - /// constraint by constraint, e.g. {dg0dx0, dg0dx1, ... ,} - /// {dg1dx0, dg1dx1, ... ,} - /// xmin[nVar] - lower bounds - /// xmax[nVar] - upper bounds - /// xval[nVar] - input/output for optimization parameters + /** + * \brief Update the optimization parameters for a constrained + * nonlinear program + * \param dfdx vector of size nVar holding the gradients of the + * objective function with respect to + * the design variables, + * $\frac{\partial F}{\partial {\bf x}_i}$ + * for each variable on this rank. + * \param gx vector of size nCon holding the values of the + * inequality constraints. Every MPI rank should + * pass in the same values here. + * \param dgdx vector of size $\textrm{nCon}\cdot\textrm{nVar}$ + * holding the gradients of the constraints in + * row-major order. For example, {dg0dx0, dg0dx1, ...,} + * {dg1dx0, dg1dx1, ..., }, ... + * \param xmin vector of size nVar holding the lower bounds on + * the design values. \p xmin and \p xmax are + * the box constraints. + * \param xmax vector of size nVar holding the upper bounds on + * the design values. \p xmin and \p xmax are + * the box constraints. + * \param xval vector of size nVar. On entry, this holds the + * value of the design variables where the objective, + * constraints, and their gradients were evaluated. + * On exit, this holds the result of the MMA iteration, + * the next design variable value to use. + * + * \details + * The caller retains ownership of all Vectors passed into this method. + */ void Update(const Vector& dfdx, const Vector& gx, const Vector& dgdx, const Vector& xmin, const Vector& xmax, Vector& xval); - /// Unconstrained + + /** + * \brief Update the optimization parameters for an unconstrained + * nonlinear program + * \param dfdx vector of size nVar holding the gradients of the + * objective function with respect to + * the design variables, + * $\frac{\partial F}{\partial {\bf x}_i}$ + * for each variable on this rank. + * \param xmin vector of size nVar holding the lower bounds on + * the design values. \p xmin and \p xmax are + * the box constraints. + * \param xmax vector of size nVar holding the upper bounds on + * the design values. \p xmin and \p xmax are + * the box constraints. + * \param xval vector of size nVar. On entry, this holds the + * value of the design variables where the objective, + * constraints, and their gradients were evaluated. + * On exit, this holds the result of the MMA iteration, + * the next design variable value to use. + * + * \details + * The caller retains ownership of all Vectors passed into this method. + * This should be used when the number of inequality constraints is zero. + */ void Update( const Vector& dfdx, const Vector& xmin, const Vector& xmax, Vector& xval); + /** + * \brief Change the iteration number + * \param iterationNumber the new iteration number + */ void SetIteration( int iterationNumber ) { iter = iterationNumber; }; - int GetIteration() { return iter; }; + /// Return the current iteration number + int GetIteration() const { return iter; }; + + /** + * \brief change the print level + * \param print_lvl the new print level + */ void SetPrintLevel(int print_lvl) { print_level = print_lvl; } protected: @@ -123,7 +258,7 @@ private: /// KKT norm real_t kktnorm; - /// intialization state + /// initialization state bool isInitialized = false; #ifdef MFEM_USE_MPI diff --git a/linalg/petsc.cpp b/linalg/petsc.cpp index 866bc5f870..2d485707c6 100644 --- a/linalg/petsc.cpp +++ b/linalg/petsc.cpp @@ -1356,6 +1356,7 @@ void PetscParMatrix::MakeWrapper(MPI_Comm comm, const Operator* op, Mat *A) PETSC_DECIDE,PETSC_DECIDE); PCHKERRQ(A,ierr); ierr = MatSetType(*A,MATSHELL); PCHKERRQ(A,ierr); ierr = MatShellSetContext(*A,(void *)op); PCHKERRQ(A,ierr); +#if PETSC_VERSION_LT(3,24,0) ierr = MatShellSetOperation(*A,MATOP_MULT, (void (*)())__mfem_mat_shell_apply); PCHKERRQ(A,ierr); @@ -1367,6 +1368,19 @@ void PetscParMatrix::MakeWrapper(MPI_Comm comm, const Operator* op, Mat *A) PCHKERRQ(A,ierr); ierr = MatShellSetOperation(*A,MATOP_DESTROY, (void (*)())__mfem_mat_shell_destroy); +#else + ierr = MatShellSetOperation(*A,MATOP_MULT, + (PetscErrorCodeFn*)__mfem_mat_shell_apply); + PCHKERRQ(A,ierr); + ierr = MatShellSetOperation(*A,MATOP_MULT_TRANSPOSE, + (PetscErrorCodeFn*)__mfem_mat_shell_apply_transpose); + PCHKERRQ(A,ierr); + ierr = MatShellSetOperation(*A,MATOP_COPY, + (PetscErrorCodeFn*)__mfem_mat_shell_copy); + PCHKERRQ(A,ierr); + ierr = MatShellSetOperation(*A,MATOP_DESTROY, + (PetscErrorCodeFn*)__mfem_mat_shell_destroy); +#endif #if defined(_USE_DEVICE) MemoryType mt = GetMemoryType(op->GetMemoryClass()); if (mt == MemoryType::DEVICE || mt == MemoryType::MANAGED) diff --git a/linalg/sparsemat.cpp b/linalg/sparsemat.cpp index ae4473c6eb..356cb7083c 100644 --- a/linalg/sparsemat.cpp +++ b/linalg/sparsemat.cpp @@ -46,6 +46,12 @@ #define MFEM_GPUSPARSE_ALG HIPSPARSE_CSRMV_ALG1 #endif // defined(MFEM_USE_CUDA) +#if defined(MFEM_USE_SINGLE) +#define MFEM_CUDA_or_HIP_REAL_T MFEM_CUDA_or_HIP(_R_32F) +#elif defined(MFEM_USE_DOUBLE) +#define MFEM_CUDA_or_HIP_REAL_T MFEM_CUDA_or_HIP(_R_64F) +#endif + namespace mfem { @@ -57,8 +63,10 @@ int SparseMatrix::SparseMatrixCount = 0; /// @cond Suppress_Doxygen_warnings MFEM_cu_or_hip(sparseHandle_t) SparseMatrix::handle = nullptr; /// @endcond +#ifndef MFEM_CUDA_1897_WORKAROUND size_t SparseMatrix::bufferSize = 0; void * SparseMatrix::dBuffer = nullptr; +#endif #endif // MFEM_USE_CUDA_OR_HIP void SparseMatrix::InitGPUSparse() @@ -464,109 +472,67 @@ void SparseMatrix::SortColumnIndices() } #ifdef MFEM_USE_CUDA_OR_HIP - if ( Device::Allows( Backend::CUDA_MASK )) + if (Device::Allows(Backend::CUDA_MASK) || Device::Allows(Backend::HIP_MASK)) { -#if defined(MFEM_USE_CUDA) - size_t pBufferSizeInBytes = 0; - void *pBuffer = NULL; - - const int n = Height(); - const int m = Width(); + const int m = Height(); + const int n = Width(); const int nnzA = J.Capacity(); - real_t * d_a_sorted = ReadWriteData(); - const int * d_ia = ReadI(); - int * d_ja_sorted = ReadWriteJ(); - csru2csrInfo_t sortInfoA; + const int *d_ia = ReadI(); + int *d_ja = ReadWriteJ(); - cusparseMatDescr_t matA_descr; - cusparseCreateMatDescr( &matA_descr ); - cusparseSetMatIndexBase( matA_descr, CUSPARSE_INDEX_BASE_ZERO ); - cusparseSetMatType( matA_descr, CUSPARSE_MATRIX_TYPE_GENERAL ); + // Get size of temporary buffer needed to sort the column indices, + // allocate the temporary buffer. + size_t pBufferSizeInBytes; + MFEM_cu_or_hip(sparseXcsrsort_bufferSizeExt)(handle, m, n, nnzA, d_ia, + d_ja, &pBufferSizeInBytes); + void *pBuffer = MFEM_Cu_or_Hip(MemAlloc)(&pBuffer, pBufferSizeInBytes); - cusparseCreateCsru2csrInfo( &sortInfoA ); + // Create matrix descriptor, will have default values + // CUSPARSE_INDEX_BASE_ZERO and CUSPARSE_MATRIX_TYPE_GENERAL. + MFEM_cu_or_hip(sparseMatDescr_t) matA_descr; + MFEM_cu_or_hip(sparseCreateMatDescr)(&matA_descr); -#ifdef MFEM_USE_SINGLE - cusparseScsru2csr_bufferSizeExt( handle, n, m, nnzA, d_a_sorted, d_ia, - d_ja_sorted, sortInfoA, - &pBufferSizeInBytes); -#elif defined MFEM_USE_DOUBLE - cusparseDcsru2csr_bufferSizeExt( handle, n, m, nnzA, d_a_sorted, d_ia, - d_ja_sorted, sortInfoA, - &pBufferSizeInBytes); -#else - MFEM_ABORT("Floating point type undefined"); -#endif + // Initialize permutation to identity + Array P(nnzA); + int *d_P = P.Write(); + mfem::forall(nnzA, [=] MFEM_HOST_DEVICE (int i) { d_P[i] = i; }); - CuMemAlloc( &pBuffer, pBufferSizeInBytes ); + // Sort the column indices. The array d_ja will now be sorted. The + // permutation required to sort the values will be returned in d_P. + MFEM_cu_or_hip(sparseXcsrsort)(handle, m, n, nnzA, matA_descr, d_ia, d_ja, + d_P, pBuffer); -#ifdef MFEM_USE_SINGLE - cusparseScsru2csr( handle, n, m, nnzA, matA_descr, d_a_sorted, d_ia, - d_ja_sorted, sortInfoA, pBuffer); -#elif defined MFEM_USE_DOUBLE - cusparseDcsru2csr( handle, n, m, nnzA, matA_descr, d_a_sorted, d_ia, - d_ja_sorted, sortInfoA, pBuffer); -#else - MFEM_ABORT("Floating point type undefined"); -#endif + // Create a copy of the unsorted matrix values. + real_t *d_a = ReadWriteData(); + void *d_a_unsorted = MFEM_Cu_or_Hip(MemAlloc)(&d_a_unsorted, + nnzA * sizeof(real_t)); + MFEM_Cu_or_Hip(MemcpyDtoD)(d_a_unsorted, d_a, nnzA * sizeof(real_t)); - // The above call is (at least in some cases) asynchronous, so we need to - // wait for it to finish before we can free device temporaries. + // Create the (input) dense vector with the unsorted values. + MFEM_cu_or_hip(sparseDnVecDescr_t) d_a_dense; + MFEM_cu_or_hip(sparseCreateDnVec)(&d_a_dense, nnzA, d_a_unsorted, + MFEM_CUDA_or_HIP_REAL_T); + + // Create the (output) sparse vector that will have the sorted values. + MFEM_cu_or_hip(sparseSpVecDescr_t) d_a_sparse; + MFEM_cu_or_hip(sparseCreateSpVec)(&d_a_sparse, nnzA, nnzA, d_P, d_a, + MFEM_CU_or_HIP(SPARSE_INDEX_32I), + MFEM_CU_or_HIP(SPARSE_INDEX_BASE_ZERO), + MFEM_CUDA_or_HIP_REAL_T); + + // Sort the matrix values using the permutation vector. + MFEM_cu_or_hip(sparseGather)(handle, d_a_dense, d_a_sparse); + + // The above calls may be asynchronous, so we need to wait for them to + // finish before we can free memory. MFEM_STREAM_SYNC; - cusparseDestroyCsru2csrInfo( sortInfoA ); - cusparseDestroyMatDescr( matA_descr ); + MFEM_cu_or_hip(sparseDestroyDnVec)(d_a_dense); + MFEM_cu_or_hip(sparseDestroySpVec)(d_a_sparse); + MFEM_cu_or_hip(sparseDestroyMatDescr)(matA_descr); - CuMemFree( pBuffer ); -#endif - } - else if ( Device::Allows( Backend::HIP_MASK )) - { -#if defined(MFEM_USE_HIP) - size_t pBufferSizeInBytes = 0; - void *pBuffer = NULL; - int *P = NULL; - - const int n = Height(); - const int m = Width(); - const int nnzA = J.Capacity(); - real_t * d_a_sorted = ReadWriteData(); - const int * d_ia = ReadI(); - int * d_ja_sorted = ReadWriteJ(); - - hipsparseMatDescr_t descrA; - hipsparseCreateMatDescr( &descrA ); - // FIXME: There is not in-place version of csr sort in hipSPARSE currently, so we make - // a temporary copy of the data for gthr, sort that, and then copy the sorted values - // back to the array being returned. Where there is an in-place version available, - // we should use it. - Array< real_t > a_tmp( nnzA ); - real_t *d_a_tmp = a_tmp.Write(); - - hipsparseXcsrsort_bufferSizeExt(handle, n, m, nnzA, d_ia, d_ja_sorted, - &pBufferSizeInBytes); - - HipMemAlloc( &pBuffer, pBufferSizeInBytes ); - HipMemAlloc( (void**)&P, nnzA * sizeof(int) ); - - hipsparseCreateIdentityPermutation(handle, nnzA, P); - hipsparseXcsrsort(handle, n, m, nnzA, descrA, d_ia, d_ja_sorted, P, pBuffer); - -#if defined(MFEM_USE_SINGLE) - hipsparseSgthr(handle, nnzA, d_a_sorted, d_a_tmp, P, - HIPSPARSE_INDEX_BASE_ZERO); -#elif defined(MFEM_USE_DOUBLE) - hipsparseDgthr(handle, nnzA, d_a_sorted, d_a_tmp, P, - HIPSPARSE_INDEX_BASE_ZERO); -#else - MFEM_ABORT("Unsupported floating point type!"); -#endif - - A.CopyFrom( a_tmp.GetMemory(), nnzA ); - hipsparseDestroyMatDescr( descrA ); - - HipMemFree( pBuffer ); - HipMemFree( P ); -#endif + MFEM_Cu_or_Hip(MemFree)(d_a_unsorted); + MFEM_Cu_or_Hip(MemFree)(pBuffer); } else #endif // MFEM_USE_CUDA_OR_HIP @@ -821,27 +787,15 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const real_t a) const MFEM_CU_or_HIP(SPARSE_INDEX_32I), MFEM_CU_or_HIP(SPARSE_INDEX_32I), MFEM_CU_or_HIP(SPARSE_INDEX_BASE_ZERO), -#ifdef MFEM_USE_SINGLE - MFEM_CUDA_or_HIP(_R_32F)); -#else - MFEM_CUDA_or_HIP(_R_64F)); -#endif + MFEM_CUDA_or_HIP_REAL_T); // Create handles for input/output vectors MFEM_cu_or_hip(sparseCreateDnVec)(&vecX_descr, x.Size(), const_cast(d_x), -#ifdef MFEM_USE_SINGLE - MFEM_CUDA_or_HIP(_R_32F)); -#else - MFEM_CUDA_or_HIP(_R_64F)); -#endif + MFEM_CUDA_or_HIP_REAL_T); MFEM_cu_or_hip(sparseCreateDnVec)(&vecY_descr, y.Size(), d_y, -#ifdef MFEM_USE_SINGLE - MFEM_CUDA_or_HIP(_R_32F)); -#else - MFEM_CUDA_or_HIP(_R_64F)); -#endif + MFEM_CUDA_or_HIP_REAL_T); #else cusparseCreateMatDescr(&matA_descr); cusparseSetMatIndexBase(matA_descr, CUSPARSE_INDEX_BASE_ZERO); @@ -860,11 +814,7 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const real_t a) const vecX_descr, &beta, vecY_descr, -#ifdef MFEM_USE_SINGLE - MFEM_CUDA_or_HIP(_R_32F), -#else - MFEM_CUDA_or_HIP(_R_64F), -#endif + MFEM_CUDA_or_HIP_REAL_T, MFEM_GPUSPARSE_ALG, &newBufferSize); @@ -891,11 +841,7 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const real_t a) const vecX_descr, &beta, vecY_descr, -#ifdef MFEM_USE_SINGLE - MFEM_CUDA_or_HIP(_R_32F), -#else - MFEM_CUDA_or_HIP(_R_64F), -#endif + MFEM_CUDA_or_HIP_REAL_T, MFEM_GPUSPARSE_ALG, dBuffer); #else @@ -4372,6 +4318,14 @@ SparseMatrix::~SparseMatrix() #ifdef MFEM_USE_CUDA_OR_HIP if (Device::Allows(Backend::CUDA_MASK | Backend::HIP_MASK)) { +#ifdef MFEM_CUDA_1897_WORKAROUND + if (dBuffer) + { + MFEM_Cu_or_Hip(MemFree)(dBuffer); + dBuffer = nullptr; + bufferSize = 0; + } +#endif if (SparseMatrixCount==1) { if (handle) @@ -4379,12 +4333,14 @@ SparseMatrix::~SparseMatrix() MFEM_cu_or_hip(sparseDestroy)(handle); handle = nullptr; } +#ifndef MFEM_CUDA_1897_WORKAROUND if (dBuffer) { MFEM_Cu_or_Hip(MemFree)(dBuffer); dBuffer = nullptr; bufferSize = 0; } +#endif } SparseMatrixCount--; } diff --git a/linalg/sparsemat.hpp b/linalg/sparsemat.hpp index 095ba5a471..b64af40fa6 100644 --- a/linalg/sparsemat.hpp +++ b/linalg/sparsemat.hpp @@ -98,9 +98,17 @@ protected: #ifdef MFEM_USE_CUDA_OR_HIP // common for hipSPARSE and cuSPARSE static int SparseMatrixCount; + mutable bool initBuffers = false; + +#if defined(MFEM_USE_CUDA) && CUDA_VERSION >= 12300 && CUDA_VERSION < 12602 + // Workaround for bug CUSPARSE-1897 +#define MFEM_CUDA_1897_WORKAROUND + mutable size_t bufferSize = 0; + mutable void *dBuffer = nullptr; +#else static size_t bufferSize; static void *dBuffer; - mutable bool initBuffers = false; +#endif #if defined(MFEM_USE_CUDA) cusparseStatus_t status; diff --git a/linalg/sundials.cpp b/linalg/sundials.cpp index 0319d20fb7..bdcd1d3eec 100644 --- a/linalg/sundials.cpp +++ b/linalg/sundials.cpp @@ -215,11 +215,23 @@ Sundials::Sundials() MFEM_VERIFY(return_val == 0, "Call to SUNContext_Create failed"); SundialsMemHelper actual_helper(context); memHelper = std::move(actual_helper); + isInitialized = true; } Sundials::~Sundials() { - SUNContext_Free(&context); + Sundials::Finalize(); +} + +void Sundials::Finalize() +{ + Sundials& sundials = Instance(); + if (sundials.isInitialized) + { + SUNContext context = GetContext(); + SUNContext_Free(&context); + sundials.isInitialized = false; + } } #else // SUNDIALS_VERSION_MAJOR >= 6 @@ -234,6 +246,11 @@ Sundials::~Sundials() // Do nothing } +void Sundials::Finalize() +{ + // Do nothing +} + #endif // SUNDIALS_VERSION_MAJOR >= 6 #if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP) diff --git a/linalg/sundials.hpp b/linalg/sundials.hpp index 7aa9c8c5f9..56292f0ec3 100644 --- a/linalg/sundials.hpp +++ b/linalg/sundials.hpp @@ -182,10 +182,19 @@ public: /// Disable copy assignment void operator=(const Sundials &other) = delete; - /// Initializes SUNContext and SundialsMemHelper objects. Should be called at + /// @brief Initializes SUNContext and SundialsMemHelper objects. Should be called at /// the beginning of the calling program (after Mpi::Init if applicable) + /// + /// Multiple calls to Sundials::Init() have no effect. static void Init(); + /// @brief Finalize sundials (called automatically at program exit if + /// Sundials::Init() has been called). + /// + /// Multiple calls to Sundials::Finalize() have no effect. This function can be + /// called manually to more precisely control when sundials is finalized. + static void Finalize(); + /// Provides access to the SUNContext object static SUNContext &GetContext(); @@ -202,6 +211,7 @@ private: /// Destructor called at end of calling program (does nothing for version < 6) ~Sundials(); + bool isInitialized; SUNContext context; SundialsMemHelper memHelper; }; diff --git a/linalg/tensor.hpp b/linalg/tensor.hpp index 41cdd71418..5ce2f6a93c 100644 --- a/linalg/tensor.hpp +++ b/linalg/tensor.hpp @@ -60,6 +60,19 @@ struct tensor T values[n0]; }; +template < typename T > +struct tensor +{ + using type = T; + static constexpr int ndim = 1; + static constexpr int first_dim = 0; + MFEM_HOST_DEVICE T& operator[](int /*unused*/) { return values; } + MFEM_HOST_DEVICE const T& operator[](int /*unused*/) const { return values; } + MFEM_HOST_DEVICE T& operator()(int /*unused*/) { return values; } + MFEM_HOST_DEVICE const T& operator()(int /*unused*/) const { return values; } + T values; +}; + template < typename T, int n0, int n1 > struct tensor { @@ -75,6 +88,21 @@ struct tensor tensor < T, n1 > values[n0]; }; +template < typename T, int n1 > +struct tensor +{ + using type = T; + static constexpr int ndim = 2; + static constexpr int first_dim = 0; + MFEM_HOST_DEVICE tensor< T, n1 >& operator[](int /*unused*/) { return values; } + MFEM_HOST_DEVICE const tensor< T, n1 >& operator[](int /*unused*/) const { return values; } + MFEM_HOST_DEVICE tensor< T, n1 >& operator()(int /*unused*/) { return values; } + MFEM_HOST_DEVICE const tensor< T, n1 >& operator()(int /*unused*/) const { return values; } + MFEM_HOST_DEVICE T& operator()(int /*unused*/, int j) { return values[j]; } + MFEM_HOST_DEVICE const T& operator()(int /*unused*/, int j) const { return values[j]; } + tensor < T, n1 > values; +}; + template < typename T, int n0, int n1, int n2 > struct tensor { @@ -1243,6 +1271,31 @@ T norm(const tensor& A) return std::sqrt(sqnorm(A)); } +template MFEM_HOST_DEVICE +T weight(const tensor& A) +{ + static_assert((n == m) || ((n == 2) && (m == 1)) || ((n == 3) && (m == 1)) || + ((n == 3) && (m == 2)), "unsupported combination of n and m"); + if constexpr (n == m) + { + return det(A); + } + if constexpr (((n == 2) && (m == 1)) || + ((n == 3) && (m == 1))) + { + return norm(A); + } + else if constexpr ((n == 3) && (m == 2)) + { + T E = A[0][0] * A[0][0] + A[1][0] * A[1][0] + A[2][0] * A[2][0]; + T G = A[0][1] * A[0][1] + A[1][1] * A[1][1] + A[2][1] * A[2][1]; + T F = A[0][0] * A[0][1] + A[1][0] * A[1][1] + A[2][0] * A[2][1]; + return std::sqrt(E * G - F * F); + } + // Never reached because of the static_assert, but avoids compiler warning. + return T{}; +} + /** * @brief Normalizes the tensor * Each element is divided by the Frobenius norm of the tensor, @see norm @@ -1518,7 +1571,7 @@ T calcsv(const tensor A, const int i) * @return Whether the square rank 2 tensor (matrix) is symmetric */ template MFEM_HOST_DEVICE -bool is_symmetric(tensor A, real_t abs_tolerance = 1.0e-8) +bool is_symmetric(tensor A, real_t abs_tolerance = 1.0e-8_r) { for (int i = 0; i < n; ++i) { @@ -1659,7 +1712,7 @@ inline MFEM_HOST_DEVICE tensor inv(const tensor& A) template inline MFEM_HOST_DEVICE tensor inv(const tensor& A) { - T inv_detA(1.0 / det(A)); + T inv_detA(1.0_r / det(A)); tensor invA{}; @@ -1678,7 +1731,7 @@ inline MFEM_HOST_DEVICE tensor inv(const tensor& A) template inline MFEM_HOST_DEVICE tensor inv(const tensor& A) { - T inv_detA(1.0 / det(A)); + T inv_detA(1.0_r / det(A)); tensor invA{}; diff --git a/linalg/ttensor.hpp b/linalg/ttensor.hpp index b7cc2e33ce..aca25143f3 100644 --- a/linalg/ttensor.hpp +++ b/linalg/ttensor.hpp @@ -281,7 +281,6 @@ inline void TAssign(const A_layout_t &A_layout, A_data_t &A_data, template Assign(A_layout, A_data, B_layout, B_data); } - // classes TVector, TMatrix, TTensor3, TTensor4 template diff --git a/linalg/vector.cpp b/linalg/vector.cpp index dcb33739e2..bfac667e98 100644 --- a/linalg/vector.cpp +++ b/linalg/vector.cpp @@ -14,6 +14,7 @@ #include "../general/forall.hpp" #include "../general/reducers.hpp" #include "../general/hash.hpp" +#include "../general/scan.hpp" #include "vector.hpp" #ifdef MFEM_USE_OPENMP @@ -117,7 +118,7 @@ Vector::Vector(const Vector &v) UseDevice(v.UseDevice()); } -Vector::Vector(Vector &&v) +Vector::Vector(Vector &&v) : Vector() { *this = std::move(v); } @@ -1252,4 +1253,33 @@ real_t Vector::Sum() const return res; } +void Vector::DeleteAt(const Array &indices) +{ + if (indices.Size()) + { + const bool use_dev = UseDevice(); + + // extra entry for number of selected out + Array workspace(size + 1); + const auto d_flag = workspace.Write(use_dev); + mfem::forall_switch(use_dev, size, + [=] MFEM_HOST_DEVICE(int i) { d_flag[i] = true; }); + const auto d_indices = indices.Read(use_dev); + mfem::forall_switch(use_dev, indices.Size(), [=] MFEM_HOST_DEVICE(int i) + { + // fine as long as indices are unique; to support non-unique indices + // assignment to d_flag must be atomic + d_flag[d_indices[i]] = false; + }); + + Vector copy(*this); + auto d_in = copy.Read(use_dev); + auto d_out = Write(use_dev); + CopyFlagged(use_dev, d_in, d_flag, d_out, d_flag + size, size); + + // assumes indices are unique + size -= indices.Size(); + } +} + } // namespace mfem diff --git a/linalg/vector.hpp b/linalg/vector.hpp index 0cf15e33e4..44bc505122 100644 --- a/linalg/vector.hpp +++ b/linalg/vector.hpp @@ -171,6 +171,13 @@ public: /// Resize the vector to size @a s using the MemoryType of @a v. void SetSize(int s, const Vector &v) { SetSize(s, v.GetMemory().GetMemoryType()); } + /// Update \ref Capacity() to @a res (if less than current), keeping existing entries. + void Reserve(int res); + + /// Delete entries at @a indices and resize vector accordingly. + /// @warning Indices must be unique! + void DeleteAt(const Array &indices); + /// Set the Vector data. /// @warning This method should be called only when OwnsData() is false. void SetData(real_t *d) { data.Wrap(d, data.Capacity(), false); } @@ -621,6 +628,18 @@ inline void Vector::SetSize(int s, MemoryType mt) data.UseDevice(use_dev); } +inline void Vector::Reserve(int res) +{ + if (res > Capacity()) + { + Memory p(res, data.GetMemoryType()); + p.CopyFrom(data, size); + p.UseDevice(data.UseDevice()); + data.Delete(); + data = p; + } +} + inline void Vector::NewMemoryAndSize(const Memory &mem, int s, bool own_mem) { diff --git a/makefile b/makefile index 37556c1aad..253b393079 100644 --- a/makefile +++ b/makefile @@ -377,7 +377,7 @@ MFEM_CONFIG_VARS = MFEM_CXX MFEM_HOST_CXX MFEM_CPPFLAGS MFEM_CXXFLAGS\ MFEM_INC_DIR MFEM_TPLFLAGS MFEM_INCFLAGS MFEM_PICFLAG MFEM_FLAGS MFEM_LIB_DIR\ MFEM_EXT_LIBS MFEM_LIBS MFEM_LIB_FILE MFEM_STATIC MFEM_SHARED MFEM_BUILD_TAG\ MFEM_PREFIX MFEM_CONFIG_EXTRA MFEM_MPIEXEC MFEM_MPIEXEC_NP MFEM_MPI_NP\ - MFEM_TEST_MK + MFEM_TEST_MK MFEM_XLINKER # Config vars: values of the form @VAL@ are replaced by $(VAL) in config.mk MFEM_CPPFLAGS ?= $(CPPFLAGS) @@ -394,6 +394,7 @@ MFEM_BUILD_TAG ?= $(shell uname -snm) MFEM_PREFIX ?= $(PREFIX) MFEM_INC_DIR ?= $(if $(CONFIG_FILE_DEF),@MFEM_BUILD_DIR@,@MFEM_DIR@) MFEM_LIB_DIR ?= $(if $(CONFIG_FILE_DEF),@MFEM_BUILD_DIR@,@MFEM_DIR@) +MFEM_XLINKER ?= $(XLINKER) MFEM_TEST_MK ?= @MFEM_DIR@/config/test.mk # Use "\n" (interpreted by sed) to add a newline. MFEM_CONFIG_EXTRA ?= $(if $(CONFIG_FILE_DEF),MFEM_BUILD_DIR ?= @MFEM_DIR@,) diff --git a/mesh/CMakeLists.txt b/mesh/CMakeLists.txt index 1a6142c56f..d0833136b7 100644 --- a/mesh/CMakeLists.txt +++ b/mesh/CMakeLists.txt @@ -20,6 +20,7 @@ set(SRCS mesh_operators.cpp mesh_readers.cpp ncmesh.cpp + ncnurbs.cpp nurbs.cpp point.cpp pyramid.cpp @@ -48,6 +49,7 @@ set(HDRS mesh_headers.hpp mesh_operators.hpp ncmesh.hpp + ncnurbs.hpp nurbs.hpp point.hpp pyramid.hpp diff --git a/mesh/mesh.cpp b/mesh/mesh.cpp index 36ef9f0a29..b0643f30b3 100644 --- a/mesh/mesh.cpp +++ b/mesh/mesh.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -921,6 +922,103 @@ const FaceGeometricFactors* Mesh::GetFaceGeometricFactors( return gf; } +const Array& Mesh::GetBdrFaceAttributes() const +{ + if (bdr_face_attrs_cache.Size() == 0) + { + std::unordered_map f_to_be; + for (int i = 0; i < GetNBE(); ++i) + { + const int f = GetBdrElementFaceIndex(i); + f_to_be[f] = i; + } + const int nf_bdr = GetNFbyType(FaceType::Boundary); + // MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes"); + bdr_face_attrs_cache.SetSize(nf_bdr); + int f_ind = 0; + const int nf = GetNumFaces(); + for (int f = 0; f < nf; ++f) + { + if (!GetFaceInformation(f).IsOfFaceType(FaceType::Boundary)) + { + continue; + } + int attribute = -1; // default value + auto iter = f_to_be.find(f); + if (iter != f_to_be.end()) + { + const int be = iter->second; + attribute = GetBdrAttribute(be); + } + else + { + // If a boundary face does not correspond to the a boundary element, + // we assign it the default attribute of -1. + } + bdr_face_attrs_cache[f_ind] = attribute; + ++f_ind; + } + } + return bdr_face_attrs_cache; +} + +const Array& Mesh::GetElementAttributes() const +{ + if (elem_attrs_cache.Size() == 0) + { + // re-compute cache + elem_attrs_cache.SetSize(GetNE()); + elem_attrs_cache.HostWrite(); + for (int i = 0; i < GetNE(); ++i) + { + elem_attrs_cache[i] = GetAttribute(i); + MFEM_ASSERT(elem_attrs_cache[i] > 0, + "Negative attribute on element " << i); + } + } + return elem_attrs_cache; +} + +void Mesh::ComputeFaceInfo(FaceType ftype) const +{ + auto &fidcs = face_indices[static_cast(ftype)]; + auto &ifidcs = inv_face_indices[static_cast(ftype)]; + fidcs.SetSize(GetNFbyType(ftype)); + fidcs.HostWrite(); + ifidcs.reserve(fidcs.Size()); + int f_idx = 0; + for (int i = 0; i < GetNumFacesWithGhost(); ++i) + { + const FaceInformation face = GetFaceInformation(i); + if (face.IsNonconformingCoarse() || !face.IsOfFaceType(ftype)) + { + continue; + } + fidcs[f_idx] = i; + ifidcs[i] = f_idx; + ++f_idx; + } +} + +const Array &Mesh::GetFaceIndices(FaceType ftype) const +{ + if (face_indices[static_cast(ftype)].Size() == 0) + { + ComputeFaceInfo(ftype); + } + return face_indices[static_cast(ftype)]; +} + +const std::unordered_map & +Mesh::GetInvFaceIndices(FaceType ftype) const +{ + if (inv_face_indices[static_cast(ftype)].empty()) + { + ComputeFaceInfo(ftype); + } + return inv_face_indices[static_cast(ftype)]; +} + void Mesh::DeleteGeometricFactors() { for (int i = 0; i < geom_factors.Size(); i++) @@ -1804,8 +1902,17 @@ void Mesh::Destroy() TetMemory.Clear(); #endif + elem_attrs_cache.DeleteAll(); + bdr_face_attrs_cache.DeleteAll(); attributes.DeleteAll(); bdr_attributes.DeleteAll(); + + face_indices[0].DeleteAll(); + face_indices[1].DeleteAll(); + // force de-allocation so after this mesh has the smallest memory footprint + // possible + inv_face_indices[0] = std::unordered_map(); + inv_face_indices[1] = std::unordered_map(); } void Mesh::ResetLazyData() @@ -1817,36 +1924,53 @@ void Mesh::ResetLazyData() DeleteGeometricFactors(); nbInteriorFaces = -1; nbBoundaryFaces = -1; + // set size to 0 so re-computations can potentially avoid a new allocation + bdr_face_attrs_cache.SetSize(0); + elem_attrs_cache.SetSize(0); + + face_indices[0].SetSize(0); + face_indices[1].SetSize(0); + inv_face_indices[0].clear(); + inv_face_indices[1].clear(); } -void Mesh::SetAttributes() +void Mesh::SetAttributes(bool elem_attrs_changed, bool bdr_face_attrs_changed) { - Array attribs; + if (bdr_face_attrs_changed) + { + bdr_face_attrs_cache.SetSize(0); // Invalidate the cache - attribs.SetSize(GetNBE()); - for (int i = 0; i < attribs.Size(); i++) - { - attribs[i] = GetBdrAttribute(i); - } - attribs.Sort(); - attribs.Unique(); - attribs.Copy(bdr_attributes); - if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0) - { - MFEM_WARNING("Non-positive attributes on the boundary!"); + // Get sorted list of unique boundary element attributes + std::set attribs; + for (int i = 0; i < GetNBE(); i++) + { + attribs.emplace(GetBdrAttribute(i)); + } + + bdr_attributes.SetSize(attribs.size()); + bdr_attributes.HostWrite(); + std::copy(attribs.begin(), attribs.end(), bdr_attributes.begin()); + if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0) + { + MFEM_WARNING("Non-positive attributes on the boundary!"); + } } - attribs.SetSize(GetNE()); - for (int i = 0; i < attribs.Size(); i++) + if (elem_attrs_changed) { - attribs[i] = GetAttribute(i); - } - attribs.Sort(); - attribs.Unique(); - attribs.Copy(attributes); - if (attributes.Size() > 0 && attributes[0] <= 0) - { - MFEM_WARNING("Non-positive attributes in the domain!"); + // Re-compute the attributes cache + elem_attrs_cache.SetSize(0); + GetElementAttributes(); + // Get sorted list of unique element attributes + std::set attribs(elem_attrs_cache.begin(), elem_attrs_cache.end()); + attributes.SetSize(attribs.size()); + attributes.HostWrite(); + std::copy(attribs.begin(), attribs.end(), attributes.begin()); + + if (attributes.Size() > 0 && attributes[0] <= 0) + { + MFEM_WARNING("Non-positive attributes in the domain!"); + } } } @@ -3372,7 +3496,6 @@ void Mesh::FinalizeHexMesh(int generate_edges, int refine, bool fix_orientation) void Mesh::FinalizeMesh(int refine, bool fix_orientation) { FinalizeTopology(); - Finalize(refine, fix_orientation); } @@ -4035,6 +4158,8 @@ void Mesh::Make3D24TetsFromHex(int nx, int ny, int nz, ind[5] = VertexIndex(x+1, y , z+1); ind[6] = VertexIndex(x+1, y+1, z+1); ind[7] = VertexIndex( x, y+1, z+1); + // *INDENT-ON* + AddHexAs24TetsWithPoints(ind, hex_face_verts, 1); } } @@ -4055,8 +4180,8 @@ void Mesh::Make3D24TetsFromHex(int nx, int ny, int nz, auto get3array = [](Array v) { - v.Sort(); - return std::array{v[0], v[1], v[2]}; + v.Sort(); + return std::array {v[0], v[1], v[2]}; }; Array el_faces; @@ -4064,32 +4189,32 @@ void Mesh::Make3D24TetsFromHex(int nx, int ny, int nz, Array vertidxs; for (int i = 0; i < el_to_face->Size(); i++) { - el_to_face->GetRow(i, el_faces); - for (int j = 0; j < el_faces.Size(); j++) - { - GetFaceVertices(el_faces[j], vertidxs); - auto t = get3array(vertidxs); - auto it = tet_face_count.find(t); - if (it == tet_face_count.end()) //edge does not already exist - { - tet_face_count.insert({t, 1}); - face_count_map.insert({t, el_faces[j]}); - } - else - { - it->second++; // increase edge count value by 1. - } - } + el_to_face->GetRow(i, el_faces); + for (int j = 0; j < el_faces.Size(); j++) + { + GetFaceVertices(el_faces[j], vertidxs); + auto t = get3array(vertidxs); + auto it = tet_face_count.find(t); + if (it == tet_face_count.end()) //edge does not already exist + { + tet_face_count.insert({t, 1}); + face_count_map.insert({t, el_faces[j]}); + } + else + { + it->second++; // increase edge count value by 1. + } + } } for (const auto &edge : tet_face_count) { - if (edge.second == 1) //if this only appears once, it is a boundary edge - { - int facenum = (face_count_map.find(edge.first))->second; - GetFaceVertices(facenum, vertidxs); - AddBdrTriangle(vertidxs, 1); - } + if (edge.second == 1) //if this only appears once, it is a boundary edge + { + int facenum = (face_count_map.find(edge.first))->second; + GetFaceVertices(facenum, vertidxs); + AddBdrTriangle(vertidxs, 1); + } } #if 0 @@ -4330,7 +4455,7 @@ void Mesh::Make1D(int n, real_t sx) } Mesh::Mesh(const Mesh &mesh, bool copy_nodes) - : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) + : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) { Dim = mesh.Dim; spaceDim = mesh.spaceDim; @@ -4454,6 +4579,10 @@ Mesh::Mesh(const Mesh &mesh, bool copy_nodes) Nodes = mesh.Nodes; own_nodes = 0; } + + // copy attribute caches + elem_attrs_cache = mesh.elem_attrs_cache; + bdr_face_attrs_cache = mesh.bdr_face_attrs_cache; } Mesh::Mesh(Mesh &&mesh) : Mesh() @@ -4506,7 +4635,7 @@ Mesh Mesh::MakeCartesian3D( } Mesh Mesh::MakeCartesian3DWith24TetsPerHex(int nx, int ny, int nz, - real_t sx, real_t sy, real_t sz) + real_t sx, real_t sy, real_t sz) { Mesh mesh; mesh.Make3D24TetsFromHex(nx, ny, nz, sx, sy, sz); @@ -4551,7 +4680,7 @@ Mesh Mesh::MakeRefined(Mesh &orig_mesh, const Array &ref_factors, Mesh::Mesh(const std::string &filename, int generate_edges, int refine, bool fix_orientation) - : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) + : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) { // Initialization as in the default constructor SetEmpty(); @@ -4570,7 +4699,7 @@ Mesh::Mesh(const std::string &filename, int generate_edges, int refine, Mesh::Mesh(std::istream &input, int generate_edges, int refine, bool fix_orientation) - : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) + : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) { SetEmpty(); Load(input, generate_edges, refine, fix_orientation); @@ -4606,7 +4735,7 @@ Mesh::Mesh(real_t *vertices_, int num_vertices, int *boundary_indices, Geometry::Type boundary_type, int *boundary_attributes, int num_boundary_elements, int dimension, int space_dimension) - : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) + : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) { if (space_dimension == -1) { @@ -4644,7 +4773,7 @@ Mesh::Mesh(real_t *vertices_, int num_vertices, } Mesh::Mesh( const NURBSExtension& ext ) - : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) + : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) { SetEmpty(); /// make an internal copy of the NURBSExtension @@ -4922,9 +5051,13 @@ void Mesh::Loader(std::istream &input, int generate_edges, { ReadNURBSMesh(input, curved, read_gf); } + else if (mesh_type == "MFEM NURBS NC-patch mesh v1.0") + { + ReadNURBSMesh(input, curved, read_gf, true, true); // Spacing is required + } else if (mesh_type == "MFEM NURBS mesh v1.1") { - ReadNURBSMesh(input, curved, read_gf, true); + ReadNURBSMesh(input, curved, read_gf, true); } else if (mesh_type == "MFEM INLINE mesh v1.0") { @@ -5032,11 +5165,24 @@ void Mesh::Loader(std::istream &input, int generate_edges, "invalid mesh: end of file tag not found"); } + if (NURBSext && NURBSext->NonconformingPatches()) + { + string ident; + skip_comment_lines(input, '#'); + // Check for the optional section "patch_cp" + if (input.peek() == 'p') + { + input >> ident; + MFEM_VERIFY(ident == "patch_cp", "Invalid mesh format"); + NURBSext->ReadCoarsePatchCP(input); + } + } + // Finalize(...) should be called after this, if needed. } Mesh::Mesh(Mesh *mesh_array[], int num_pieces) - : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) + : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) { int i, j, ie, ib, iv, *v, nv; Element *el; @@ -5763,14 +5909,15 @@ Array Mesh::MakeSimplicial_(const Mesh &orig_mesh, int *vglobal) } -void Mesh::MakeHigherOrderSimplicial_(const Mesh &orig_mesh, const Array &parent_elements) +void Mesh::MakeHigherOrderSimplicial_(const Mesh &orig_mesh, + const Array &parent_elements) { // Higher order associated to vertices are unchanged, and those for // previously existing edges. DOFs associated to new elements need to be set. const int sdim = orig_mesh.SpaceDimension(); auto *orig_fespace = orig_mesh.GetNodes()->FESpace(); SetCurvature(orig_fespace->GetMaxElementOrder(), orig_fespace->IsDGSpace(), - orig_mesh.SpaceDimension(), orig_fespace->GetOrdering()); + orig_mesh.SpaceDimension(), orig_fespace->GetOrdering()); // The dofs associated with vertices are unchanged, but there can be new dofs // associated to edges, faces and volumes. Additionally, because we know that @@ -5793,7 +5940,8 @@ void Mesh::MakeHigherOrderSimplicial_(const Mesh &orig_mesh, const Array &p // of child element DenseMatrix shape; // ndof_coarse x nnode_refined. DenseMatrix point_matrix; // sdim x nnode_refined - IntegrationRule child_nodes_in_parent; // The parent nodes that correspond to the child nodes + IntegrationRule + child_nodes_in_parent; // The parent nodes that correspond to the child nodes for (int i = 0; i < parent_elements.Size(); i++) { const int ip = parent_elements[i]; @@ -5811,72 +5959,77 @@ void Mesh::MakeHigherOrderSimplicial_(const Mesh &orig_mesh, const Array &p case Geometry::Type::PRISM : // fall through case Geometry::Type::PYRAMID : // fall through case Geometry::Type::SQUARE : + { + // Extract the vertices of parent and child, can then form the + // map from child reference coordinates to parent reference + // coordinates. Exploit the fact that for Nodes, the vertex + // entries come first, and their indexing matches the vertex + // numbering. Thus we have already have an inverse index map. + orig_mesh.GetElementVertices(ip, parent_vertices); + GetElementVertices(i, child_vertices); + node_map.SetSize(0); + for (auto cv : child_vertices) + for (int ipv = 0; ipv < parent_vertices.Size(); ipv++) + if (cv == parent_vertices[ipv]) + { + node_map.Append(ipv); + break; + } + MFEM_ASSERT(node_map.Size() == Geometry::NumVerts[GetElementBaseGeometry(i)], + "!"); + // node_map now says which of the parent vertex nodes map to each + // of the child vertex nodes. Using this can build a basis in the + // parent element from child Node values, exploit the linearity + // to then transform all nodes. + child_nodes_in_parent.SetSize(0); + const auto *orig_FE = orig_mesh.GetNodes()->FESpace()->GetFE(ip); + for (auto pn : node_map) { - // Extract the vertices of parent and child, can then form the - // map from child reference coordinates to parent reference - // coordinates. Exploit the fact that for Nodes, the vertex - // entries come first, and their indexing matches the vertex - // numbering. Thus we have already have an inverse index map. - orig_mesh.GetElementVertices(ip, parent_vertices); - GetElementVertices(i, child_vertices); - node_map.SetSize(0); - for (auto cv : child_vertices) - for (int ipv = 0; ipv < parent_vertices.Size(); ipv++) - if (cv == parent_vertices[ipv]) - { - node_map.Append(ipv); - break; - } - MFEM_ASSERT(node_map.Size() == Geometry::NumVerts[GetElementBaseGeometry(i)], "!"); - // node_map now says which of the parent vertex nodes map to each - // of the child vertex nodes. Using this can build a basis in the - // parent element from child Node values, exploit the linearity - // to then transform all nodes. - child_nodes_in_parent.SetSize(0); - const auto *orig_FE = orig_mesh.GetNodes()->FESpace()->GetFE(ip); - for (auto pn : node_map) - { - child_nodes_in_parent.Append(orig_FE->GetNodes()[pn]); - } - const auto *simplex_FE = GetNodes()->FESpace()->GetFE(i); - shape.SetSize(orig_FE->GetDof(), simplex_FE->GetDof()); // One set of evaluations per simplex dof. - Vector col; - for (int j = 0; j < simplex_FE->GetNodes().Size(); j++) - { - const auto &simplex_node = simplex_FE->GetNodes()[j]; - IntegrationPoint simplex_node_in_orig; - // Handle the 2D vs 3D case by multiplying .z by zero. - simplex_node_in_orig.Set3( - child_nodes_in_parent[0].x + - simplex_node.x * (child_nodes_in_parent[1].x - child_nodes_in_parent[0].x) - + simplex_node.y * (child_nodes_in_parent[2].x - child_nodes_in_parent[0].x) - + simplex_node.z * (child_nodes_in_parent[(sdim > 2) ? 3 : 0].x - child_nodes_in_parent[0].x), - child_nodes_in_parent[0].y + - simplex_node.x * (child_nodes_in_parent[1].y - child_nodes_in_parent[0].y) - + simplex_node.y * (child_nodes_in_parent[2].y - child_nodes_in_parent[0].y) - + simplex_node.z * (child_nodes_in_parent[(sdim > 2) ? 3 : 0].y - child_nodes_in_parent[0].y), - child_nodes_in_parent[0].z + - simplex_node.x * (child_nodes_in_parent[1].z - child_nodes_in_parent[0].z) - + simplex_node.y * (child_nodes_in_parent[2].z - child_nodes_in_parent[0].z) - + simplex_node.z * (child_nodes_in_parent[(sdim > 2) ? 3 : 0].z - child_nodes_in_parent[0].z)); - shape.GetColumnReference(j, col); - orig_FE->CalcShape(simplex_node_in_orig, col); - } - // All the non-simplex basis functions have now been evaluated at - // all the simplex basis function node locations. Now evaluate - // the summations and place back into the Nodes vector. - orig_mesh.GetNodes()->GetElementDofValues(ip, edofvals); - // Dof values are always returned as - // [[x_1,x_2,x_3,...], - // [y_1,y_2,y_3,...], - // [z_1,z_2,z_3,...]] - DenseMatrix edofvals_mat(edofvals.GetData(), orig_FE->GetDof(), sdim); - point_matrix.SetSize(simplex_FE->GetDof(), sdim); - MultAtB(shape, edofvals_mat, point_matrix); - GetNodes()->FESpace()->GetElementVDofs(i, edofs); - GetNodes()->SetSubVector(edofs, point_matrix.GetData()); + child_nodes_in_parent.Append(orig_FE->GetNodes()[pn]); } - break; + const auto *simplex_FE = GetNodes()->FESpace()->GetFE(i); + shape.SetSize(orig_FE->GetDof(), + simplex_FE->GetDof()); // One set of evaluations per simplex dof. + Vector col; + for (int j = 0; j < simplex_FE->GetNodes().Size(); j++) + { + const auto &simplex_node = simplex_FE->GetNodes()[j]; + IntegrationPoint simplex_node_in_orig; + // Handle the 2D vs 3D case by multiplying .z by zero. + simplex_node_in_orig.Set3( + child_nodes_in_parent[0].x + + simplex_node.x * (child_nodes_in_parent[1].x - child_nodes_in_parent[0].x) + + simplex_node.y * (child_nodes_in_parent[2].x - child_nodes_in_parent[0].x) + + simplex_node.z * (child_nodes_in_parent[(sdim > 2) ? 3 : 0].x - + child_nodes_in_parent[0].x), + child_nodes_in_parent[0].y + + simplex_node.x * (child_nodes_in_parent[1].y - child_nodes_in_parent[0].y) + + simplex_node.y * (child_nodes_in_parent[2].y - child_nodes_in_parent[0].y) + + simplex_node.z * (child_nodes_in_parent[(sdim > 2) ? 3 : 0].y - + child_nodes_in_parent[0].y), + child_nodes_in_parent[0].z + + simplex_node.x * (child_nodes_in_parent[1].z - child_nodes_in_parent[0].z) + + simplex_node.y * (child_nodes_in_parent[2].z - child_nodes_in_parent[0].z) + + simplex_node.z * (child_nodes_in_parent[(sdim > 2) ? 3 : 0].z - + child_nodes_in_parent[0].z)); + shape.GetColumnReference(j, col); + orig_FE->CalcShape(simplex_node_in_orig, col); + } + // All the non-simplex basis functions have now been evaluated at + // all the simplex basis function node locations. Now evaluate + // the summations and place back into the Nodes vector. + orig_mesh.GetNodes()->GetElementDofValues(ip, edofvals); + // Dof values are always returned as + // [[x_1,x_2,x_3,...], + // [y_1,y_2,y_3,...], + // [z_1,z_2,z_3,...]] + DenseMatrix edofvals_mat(edofvals.GetData(), orig_FE->GetDof(), sdim); + point_matrix.SetSize(simplex_FE->GetDof(), sdim); + MultAtB(shape, edofvals_mat, point_matrix); + GetNodes()->FESpace()->GetElementVDofs(i, edofs); + GetNodes()->SetSubVector(edofs, point_matrix.GetData()); + } + break; case Geometry::Type::POINT : // fall through case Geometry::Type::INVALID : case Geometry::Type::NUM_GEOMETRIES : @@ -6159,6 +6312,11 @@ void Mesh::KnotRemove(Array &kv) UpdateNURBS(); } +void Mesh::RefineNURBSWithKVFactors(int rf, const std::string &kvf) +{ + RefineNURBS(true, 0.0, Array(&rf, 1), kvf); +} + void Mesh::NURBSUniformRefinement(int rf, real_t tol) { Array rf_array(Dim); @@ -6171,8 +6329,13 @@ void Mesh::NURBSUniformRefinement(Array const& rf, real_t tol) MFEM_VERIFY(rf.Size() == Dim, "Refinement factors must be defined for each dimension"); - MFEM_VERIFY(NURBSext, "NURBSUniformRefinement is only for NURBS meshes"); + RefineNURBS(false, tol, rf, ""); +} +void Mesh::RefineNURBS(bool usingKVF, real_t tol, const Array &rf, + const std::string &kvf) +{ + MFEM_VERIFY(NURBSext, "This type of refinement is only for NURBS meshes"); NURBSext->ConvertToPatches(*Nodes); Array cf; @@ -6184,17 +6347,19 @@ void Mesh::NURBSUniformRefinement(Array const& rf, real_t tol) cf1 = (cf1 && f == 1); } - if (cf1) + if (!cf1 && NURBSext->NonconformingPatches()) { - NURBSext->UniformRefinement(rf); + NURBSext->FullyCoarsen(); + last_operation = Mesh::NONE; // FiniteElementSpace::Update is not supported } - else + else if (!cf1 && !NURBSext->NonconformingPatches()) { + MFEM_VERIFY(!usingKVF, "This refinement type is not supported for this" + " NURBS mesh type"); NURBSext->Coarsen(cf, tol); last_operation = Mesh::NONE; // FiniteElementSpace::Update is not supported sequence++; - UpdateNURBS(); NURBSext->ConvertToPatches(*Nodes); @@ -6202,6 +6367,18 @@ void Mesh::NURBSUniformRefinement(Array const& rf, real_t tol) NURBSext->UniformRefinement(cf); } + if (cf1 || NURBSext->NonconformingPatches()) + { + if (usingKVF || NURBSext->NonconformingPatches()) + { + NURBSext->RefineWithKVFactors(rf[0], kvf, !cf1); + } + else + { + NURBSext->UniformRefinement(rf); + } + } + last_operation = Mesh::NONE; // FiniteElementSpace::Update is not supported sequence++; @@ -6417,7 +6594,7 @@ void Mesh::GetEdgeToUniqueKnotvector(Array &edge_to_ukv, { const int ri = get_root(i); const int rj = get_root(j); - if (ri == rj) return; + if (ri == rj) { return; } // keep the lowest index (ri < rj) ? pkv_map[rj] = ri : pkv_map[ri] = rj; }; @@ -6478,6 +6655,52 @@ void Mesh::GetEdgeToUniqueKnotvector(Array &edge_to_ukv, } } +void Mesh::LoadNonconformingPatchTopo(std::istream &input, + Array &edge_to_ukv) +{ + SetEmpty(); + + // Read MFEM NURBS NC-patch mesh v1.0 format + int curved = 0; + int is_nc = 1; + + ncmesh = new NCMesh(input, 10, curved, is_nc); + + InitFromNCMesh(*ncmesh); + + skip_comment_lines(input, '#'); + + string ident; + int inputNumOfEdges = -1; + + input >> ident; // 'edges' + input >> inputNumOfEdges; + + MFEM_VERIFY(NumOfEdges == inputNumOfEdges, ""); + + edge_to_ukv.SetSize(NumOfEdges); + for (int j = 0; j < NumOfEdges; j++) + { + int v[2]; // Vertex indices + int ukv; // Unique KnotVector index + input >> ukv >> v[0] >> v[1]; + + for (int i=0; i<2; ++i) + { + v[i] = ncmesh->vertex_nodeId[v[i]]; + } + + if (v[0] > v[1]) + { + ukv = -1 - ukv; + } + edge_to_ukv[j] = ukv; + } + + FinalizeTopology(); + CheckBdrElementOrientation(); // check and fix boundary element orientation +} + void XYZ_VectorFunction(const Vector &p, Vector &v) { if (p.Size() >= v.Size()) @@ -7720,8 +7943,14 @@ void Mesh::GetBdrElementAdjacentElement2( void Mesh::SetAttribute(int i, int attr) { - elements[i]->SetAttribute(attr); - if (ncmesh) ncmesh->SetAttribute(i, attr); + elements[i]->SetAttribute(attr); + if (elem_attrs_cache.Size() == GetNE()) + { + // update the existing cache instead of deleting it + elem_attrs_cache.HostReadWrite(); + elem_attrs_cache[i] = attr; + } + if (ncmesh) { ncmesh->SetAttribute(i, attr); } } Element::Type Mesh::GetElementType(int i) const @@ -8055,6 +8284,12 @@ void Mesh::GenerateFaces() FreeElement(f); } + // delete caches + face_indices[0].SetSize(0); + face_indices[1].SetSize(0); + inv_face_indices[0].clear(); + inv_face_indices[1].clear(); + // (re)generate the interior faces and the info for them faces.SetSize(nfaces); faces_info.SetSize(nfaces); @@ -10784,7 +11019,7 @@ void Mesh::InitFromNCMesh(const NCMesh &ncmesh_) } Mesh::Mesh(const NCMesh &ncmesh_) - : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) + : attribute_sets(attributes), bdr_attribute_sets(bdr_attributes) { Init(); InitTables(); @@ -10850,6 +11085,15 @@ void Mesh::Swap(Mesh& other, bool non_geometry) mfem::Swap(nodes_sequence, other.nodes_sequence); mfem::Swap(last_operation, other.last_operation); } + + // copy attribute caches + mfem::Swap(elem_attrs_cache, other.elem_attrs_cache); + mfem::Swap(bdr_face_attrs_cache, other.bdr_face_attrs_cache); + + mfem::Swap(face_indices[0], other.face_indices[0]); + mfem::Swap(face_indices[1], other.face_indices[1]); + inv_face_indices[0].swap(other.inv_face_indices[0]); + inv_face_indices[1].swap(other.inv_face_indices[1]); } void Mesh::GetElementData(const Array &elem_array, int geom, @@ -11735,6 +11979,7 @@ void Mesh::Printer(std::ostream &os, std::string section_delimiter, os << '\n'; Nodes->Save(os); + NURBSext->PrintCoarsePatches(os); // patch-wise format // NURBSext->ConvertToPatches(*Nodes); // NURBSext->Print(os); @@ -11769,10 +12014,10 @@ void Mesh::Printer(std::ostream &os, std::string section_delimiter, // serial/parallel conforming mesh format const bool set_names = attribute_sets.SetsExist() || - bdr_attribute_sets.SetsExist(); + bdr_attribute_sets.SetsExist(); os << (!set_names && section_delimiter.empty() ? "MFEM mesh v1.0\n" : - (!set_names ? "MFEM mesh v1.2\n" : "MFEM mesh v1.3\n")); + (!set_names ? "MFEM mesh v1.2\n" : "MFEM mesh v1.3\n")); if (set_names && section_delimiter.empty()) { @@ -11804,8 +12049,8 @@ void Mesh::Printer(std::ostream &os, std::string section_delimiter, if (set_names) { - os << "\nattribute_sets\n"; - attribute_sets.Print(os); + os << "\nattribute_sets\n"; + attribute_sets.Print(os); } os << "\nboundary\n" << NumOfBdrElements << '\n'; @@ -11816,8 +12061,8 @@ void Mesh::Printer(std::ostream &os, std::string section_delimiter, if (set_names) { - os << "\nbdr_attribute_sets\n"; - bdr_attribute_sets.Print(os); + os << "\nbdr_attribute_sets\n"; + bdr_attribute_sets.Print(os); } os << "\nvertices\n" << NumOfVertices << '\n'; @@ -11849,9 +12094,9 @@ void Mesh::Printer(std::ostream &os, std::string section_delimiter, } void Mesh::PrintTopo(std::ostream &os, const Array &e_to_k, - const int version, const std::string &comments) const + const int version, const std::string &comments) const { - MFEM_VERIFY(version == 10 || version == 11, "Invalid NURBS mesh version"); + MFEM_VERIFY(version == 10 || version == 11, "Invalid NURBS mesh version"); int i; Array vert; @@ -11881,8 +12126,16 @@ void Mesh::PrintTopo(std::ostream &os, const Array &e_to_k, PrintElement(boundary[i], os); } + PrintTopoEdges(os, e_to_k); +} + +void Mesh::PrintTopoEdges(std::ostream &os, const Array &e_to_k, + bool vmap) const +{ + Array vert; + os << "\nedges\n" << NumOfEdges << '\n'; - for (i = 0; i < NumOfEdges; i++) + for (int i = 0; i < NumOfEdges; i++) { edge_vertex->GetRow(i, vert); int ki = e_to_k[i]; @@ -11890,9 +12143,30 @@ void Mesh::PrintTopo(std::ostream &os, const Array &e_to_k, { ki = -1 - ki; } + + if (vmap) + { + for (int j=0; j<2; ++j) + { + vert[j] = ncmesh->vertex_nodeId[vert[j]]; + } + + if (e_to_k[i] < 0) + { + // Swap the entries of vert + const int s = vert[0]; + vert[0] = vert[1]; + vert[1] = s; + } + } + os << ki << ' ' << vert[0] << ' ' << vert[1] << '\n'; } - os << "\nvertices\n" << NumOfVertices << '\n'; + + if (!vmap) + { + os << "\nvertices\n" << NumOfVertices << '\n'; + } } void Mesh::Save(const std::string &fname, int precision) const @@ -12079,7 +12353,7 @@ void Mesh::PrintVTU(std::string fname, VTKFormat format, bool high_order_output, int compression_level, - bool bdr) + bool bdr_elements) { int ref = (high_order_output && Nodes) ? Nodes->FESpace()->GetMaxElementOrder() : 1; @@ -12093,7 +12367,7 @@ void Mesh::PrintVTU(std::string fname, } os << " byte_order=\"" << VTKByteOrder() << "\">\n"; os << "\n"; - PrintVTU(os, ref, format, high_order_output, compression_level, bdr); + PrintVTU(os, ref, format, high_order_output, compression_level, bdr_elements); os << "\n"; // need to close the piece open in the PrintVTU method os << "\n"; os << "" << std::endl; @@ -15172,6 +15446,20 @@ Mesh *Extrude2D(Mesh *mesh, const int nz, const real_t sz) return mesh3d; } +bool Mesh::Conforming() const +{ + if (NURBSext) + { + // NURBS meshes are always conforming (element-wise). NURBS patch + // conformity is indicated by NURBSExtension::NonconformingPatches. + return true; + } + else + { + return ncmesh == NULL; + } +} + #ifdef MFEM_DEBUG void Mesh::DebugDump(std::ostream &os) const { diff --git a/mesh/mesh.hpp b/mesh/mesh.hpp index 6ec810685e..5672909ed6 100644 --- a/mesh/mesh.hpp +++ b/mesh/mesh.hpp @@ -65,6 +65,7 @@ class Mesh { friend class NCMesh; friend class NURBSExtension; + friend class NCNURBSExtension; #ifdef MFEM_USE_MPI friend class ParMesh; friend class ParNCMesh; @@ -107,6 +108,11 @@ protected: Array boundary; Array faces; + /// internal cache for element attributes + mutable Array elem_attrs_cache; + /// internal cache for boundary element attributes + mutable Array bdr_face_attrs_cache; + /** @brief This structure stores the low level information necessary to interpret the configuration of elements on a specific face. This information can be accessed using methods like GetFaceElements(), @@ -273,6 +279,13 @@ protected: // used during NC mesh initialization only Array > tmp_vertex_parents; + /// cache for FaceIndices(ftype) + mutable Array face_indices[2]; + /// cache for FaceIndices(ftype) + mutable std::unordered_map inv_face_indices[2]; + + /// compute face_indices[ftype] and inv_face_indices[type] + void ComputeFaceInfo(FaceType ftype) const; public: typedef Geometry::Constants seg_t; @@ -307,6 +320,11 @@ public: // (true) is set in mesh_readers.cpp. static bool remove_unused_vertices; + /// Map from boundary or interior face indices to mesh face indices. + const Array& GetFaceIndices(FaceType ftype) const; + /// Inverse of the map FaceIndices(ftype) + const std::unordered_map& GetInvFaceIndices(FaceType ftype) const; + protected: Operation last_operation; @@ -342,7 +360,7 @@ protected: void ReadXML_VTKMesh(std::istream &input, int &curved, int &read_gf, bool &finalize_topo, const std::string &xml_prefix=""); void ReadNURBSMesh(std::istream &input, int &curved, int &read_gf, - bool spacing=false); + bool spacing=false, bool nc=false); void ReadInlineMesh(std::istream &input, bool generate_edges = false); void ReadGmshMesh(std::istream &input, int &curved, int &read_gf); @@ -475,8 +493,24 @@ protected: /// Read NURBS patch/macro-element mesh void LoadPatchTopo(std::istream &input, Array &edge_to_ukv); + /// Read NURBS patch/macro-element mesh (MFEM NURBS NC-patch mesh format) + void LoadNonconformingPatchTopo(std::istream &input, + Array &edge_to_ukv); + + /// Update this NURBS Mesh and its NURBS data structures after a change, such + /// as refinement, derefinement, or degree change. void UpdateNURBS(); + /** @brief Refine the NURBS mesh with default refinement factors in @a rf for + each dimension. + + Optionally, if @a usingKVF is true, use refinement factors specified for + particular KnotVectors, from the file with name in @a kvf. When + coarsening by knot removal is necessary for non-nested spacing formulas, + tolerance @a tol is used (see NURBSPatch::KnotRemove()). */ + void RefineNURBS(bool usingKVF, real_t tol, const Array &rf, + const std::string &kvf); + /** @brief Write the beginning of a NURBS mesh to @a os, specifying the NURBS patch topology. Optional file comments can be provided in @a comments. @@ -489,6 +523,10 @@ protected: const int version, const std::string &comment = "") const; + /// Write the patch topology edges of a NURBS mesh (see PrintTopo()). + void PrintTopoEdges(std::ostream &out, const Array &e_to_k, + bool vmap = false) const; + /// Used in GetFaceElementTransformations (...) void GetLocalPtToSegTransformation(IsoparametricTransformation &, int i) const; @@ -1122,13 +1160,14 @@ public: Mesh vertices or nodes are set. */ virtual void Finalize(bool refine = false, bool fix_orientation = false); - /// @brief Determine the sets of unique attribute values in domain and - /// boundary elements. + /// @brief Determine the sets of unique attribute values in domain if @a + /// elem_attrs_changed and boundary elements if @a bdr_face_attrs_changed. /// /// Separately scan the domain and boundary elements to generate unique, /// sorted sets of the element attribute values present in the mesh and /// store these in the Mesh::attributes and Mesh::bdr_attributes arrays. - virtual void SetAttributes(); + virtual void SetAttributes(bool elem_attrs_changed = true, + bool bdr_face_attrs_changed = true); /// Check (and optionally attempt to fix) the orientation of the elements /** @param[in] fix_it If `true`, attempt to fix the orientations of some @@ -2267,6 +2306,35 @@ public: method does not modify the nodes. */ void NodesUpdated() { DeleteGeometricFactors(); } + /// @brief Returns the attributes for all elements in this mesh. The i'th + /// entry of the array is the attribute of the i'th element of the mesh. + /// + /// The returned array points to an internal object that may be invalidated + /// by mesh operations such as refinement or any element attributes are + /// modified. Since not all such modifications can be tracked by the Mesh + /// class (e.g. if a user calls GetElement() then changes the element + /// attribute directly), one needs to account for such changes by calling the + /// method SetAttributes(). + const Array& GetElementAttributes() const; + + /// @brief Returns the attributes for all boundary elements in this mesh. + /// + /// The face restriction will give "face E-vectors" on the boundary that + /// are numbered in the order of the faces of mesh. This numbering will be + /// different than the numbering of the boundary elements. We compute + /// mappings so that the array `bdr_attributes[i]` gives the boundary + /// attribute of the `i`th boundary face in the mesh face order. + /// Attributes <= 0 indicate there is no boundary element and should be + /// skipped. + /// + /// The returned array points to an internal object that may be invalidated + /// by mesh operations such as refinement or any element attributes are + /// modified. Since not all such modifications can be tracked by the Mesh + /// class (e.g. if a user calls GetElement() then changes the element + /// attribute directly), one needs to account for such changes by calling the + /// method SetAttributes(). + const Array& GetBdrFaceAttributes() const; + /// @} /// @anchor mfem_Mesh_gf_nodes @@ -2363,6 +2431,10 @@ public: virtual void NURBSUniformRefinement(int rf = 2, real_t tol = 1.0e-12); virtual void NURBSUniformRefinement(const Array &rf, real_t tol=1.e-12); + /** @a brief Use knotvector refinement factors loaded from the file with name + in @a kvf. Everywhere else, use the default refinement factor @a rf. */ + virtual void RefineNURBSWithKVFactors(int rf, const std::string &kvf); + /// Coarsening for a NURBS mesh, with an optional coarsening factor @a cf > 1 /// which divides the number of elements in each dimension. void NURBSCoarsening(int cf = 2, real_t tol = 1.0e-12); @@ -2418,10 +2490,8 @@ public: (default) or nonconforming. */ void EnsureNCMesh(bool simplices_nonconforming = false); - /// Return a bool indicating whether this mesh is conforming. - bool Conforming() const { return ncmesh == NULL; } - /// Return a bool indicating whether this mesh is nonconforming. - bool Nonconforming() const { return ncmesh != NULL; } + bool Conforming() const; + bool Nonconforming() const { return !Conforming(); } /** Designate this mesh for output as "NC mesh v1.1", meaning it is nonconforming with nonuniform refinement spacings. */ @@ -2525,7 +2595,7 @@ public: VTKFormat format=VTKFormat::ASCII, bool high_order_output=false, int compression_level=0, - bool bdr=false); + bool bdr_elements=false); /** Print the boundary elements of the mesh in VTU format, and output the boundary attributes as a data array (useful for boundary conditions). */ void PrintBdrVTU(std::string fname, diff --git a/mesh/mesh_readers.cpp b/mesh/mesh_readers.cpp index 14cb7d3838..bd97397d4b 100644 --- a/mesh/mesh_readers.cpp +++ b/mesh/mesh_readers.cpp @@ -10,6 +10,7 @@ // CONTRIBUTING.md for details. #include "mesh_headers.hpp" +#include "ncnurbs.hpp" #include "../fem/fem.hpp" #include "../general/binaryio.hpp" #include "../general/text.hpp" @@ -772,7 +773,7 @@ struct BufferReader : BufferReaderBase int header_entry_size = HeaderEntrySize(); int nblocks = ReadHeaderEntry(header_buf); header_buf += header_entry_size; - std::vector header(nblocks + 2); + std::vector header(nblocks + 2); for (int i=0; iDimension(); NumOfVertices = NURBSext->GetNV(); diff --git a/mesh/ncmesh.cpp b/mesh/ncmesh.cpp index 23e354cf4e..0c17d7b373 100644 --- a/mesh/ncmesh.cpp +++ b/mesh/ncmesh.cpp @@ -827,7 +827,9 @@ void NCMesh::ForceRefinement(int vn1, int vn2, int vn3, int vn4) Face* face = faces.Find(vn1, vn2, vn3, vn4); if (!face) { return; } - int elem = face->GetSingleElement(); + MFEM_VERIFY(!IsParallel(), "ForceRefinement is supported only in serial"); + + const int elem = face->GetSingleElement(); Element &el = elements[elem]; MFEM_ASSERT(!el.ref_type, "element already refined."); @@ -5102,6 +5104,55 @@ void NCMesh::GetPointMatrix(Geometry::Type geom, const char* ref_path, } } +void RemapKnotIndex(bool rev, const Array &rf, int &k); +std::pair QuadrupleToPair(const std::array &q); + +void NCMesh::RefineVertexToKnotSpan(const std::vector> &kvf, + const Array &kvext, + std::map, + std::array> &parentToKV) +{ + // Note that entries 1 and 2 of vertex_to_knotspan are (k1, k2), which are knot + // span (element) indices in the two dimensions of a patch face. + + for (int i=0; i ks; + std::array pv; + vertex_to_knotspan.GetVertex3D(i, tv, ks, pv); + + bool edgeReverse[2]; + for (int j=0; j<2; ++j) + { + const bool ascending = pv[j+1] > pv[j]; + edgeReverse[j] = !ascending; + } + + // The parent face is defined with vertices (pv0, pv1, pv2, pv3). + const std::pair parentPair = QuadrupleToPair(pv); + const std::array kv = parentToKV.at(parentPair); + RemapKnotIndex(edgeReverse[0], kvf[kv[0]], ks[0]); + RemapKnotIndex(edgeReverse[1], kvf[kv[1]], ks[1]); + vertex_to_knotspan.SetKnotSpans3D(i, ks); + } + else // 2D + { + int tv, ks; + std::array pv; + vertex_to_knotspan.GetVertex2D(i, tv, ks, pv); + const bool rev = pv[1] < pv[0]; + const std::pair parentPair(rev ? pv[1] : pv[0], rev ? pv[0] : pv[1]); + const std::array kv = parentToKV.at(parentPair); + const int kvId = kv[0]; + RemapKnotIndex(rev, kvf[kvId], ks); + vertex_to_knotspan.SetKnotSpan2D(i, ks); + } + } +} + void NCMesh::MarkCoarseLevel() { coarse_elements.SetSize(leaf_elements.Size()); @@ -6114,6 +6165,60 @@ void NCMesh::LoadVertexParents(std::istream &input) } } +void NCMesh::LoadVertexToKnotSpan(std::istream &input) +{ + if (Dim == 2) { LoadVertexToKnotSpan2D(input); } + else { LoadVertexToKnotSpan3D(input); } +} + +void NCMesh::LoadVertexToKnotSpan2D(std::istream &input) +{ + int nv; + input >> nv; + MFEM_VERIFY(0 <= nv, "Invalid vertex-to-knot data"); + vertex_to_knotspan.SetSize(2, nv); + for (int i=0; i pv; + input >> id >> ks >> pv[0] >> pv[1]; + + const bool idsExist = nodes.IdExists(id) && nodes.IdExists(pv[0]) + && nodes.IdExists(pv[1]); + + MFEM_VERIFY(idsExist && 0 < ks, "Invalid index"); + vertex_to_knotspan.SetVertex2D(i, id, ks, pv); + } +} + +void NCMesh::LoadVertexToKnotSpan3D(std::istream &input) +{ + int nv; + input >> nv; + MFEM_VERIFY(0 <= nv, "Invalid vertex-to-knot data"); + vertex_to_knotspan.SetSize(3, nv); + for (int i=0; i ks; + std::array pv; // Parent vertex indices + input >> id >> ks[0] >> ks[1] >> pv[0] >> pv[1] >> pv[2] >> pv[3]; + +#ifdef MFEM_DEBUG + bool idsExist = nodes.IdExists(id); + for (int j=0; j<4; ++j) + { + idsExist = idsExist && nodes.IdExists(pv[j]); + } + + const bool validKnotIds = (0 <= ks[0] || 0 <= ks[1]) && + (0 < ks[0] || 0 < ks[1]); + MFEM_ASSERT(idsExist && validKnotIds, "Invalid index"); +#endif + vertex_to_knotspan.SetVertex3D(i, id, ks, pv); + } +} + int NCMesh::PrintBoundary(std::ostream *os) const { static const int nfv2geom[5] = @@ -6241,9 +6346,14 @@ bool NCMesh::ZeroRootStates() const return true; } -void NCMesh::Print(std::ostream &os, const std::string &comments) const +void NCMesh::Print(std::ostream &os, const std::string &comments, + bool nurbs) const { - if (using_scaling) + if (nurbs) + { + os << "MFEM NURBS NC-patch mesh v1.0\n\n"; + } + else if (using_scaling) { os << "MFEM NC mesh v1.1\n\n"; } @@ -6319,6 +6429,12 @@ void NCMesh::Print(std::ostream &os, const std::string &comments) const } } + if (nurbs && vertex_to_knotspan.Size() > 0) + { + os << "\nvertex_to_knotspan\n"; + vertex_to_knotspan.Print(os); + } + if (coordinates.Size()) { os << "\n# top-level node coordinates"; @@ -6511,6 +6627,15 @@ NCMesh::NCMesh(std::istream &input, int version, int &curved, int &is_nc) input >> ident; } + // load map from hanging patch vertices to patch edge knots + if (ident == "vertex_to_knotspan") + { + LoadVertexToKnotSpan(input); + + skip_comment_lines(input, '#'); + input >> ident; + } + // load root states if (ident == "root_state") { diff --git a/mesh/ncmesh.hpp b/mesh/ncmesh.hpp index f5f8f50ea8..b35661e311 100644 --- a/mesh/ncmesh.hpp +++ b/mesh/ncmesh.hpp @@ -114,7 +114,57 @@ void Swap(CoarseFineTransformations &a, CoarseFineTransformations &b); struct MatrixMap; // for internal use -/** \brief A class for non-conforming AMR. The class is not used directly by the +/** @brief For a NURBS mesh with nonconforming patch topology, this struct + provides a map from hanging vertices in the patch topology to the knotvector + of a neighboring patch. This facilitates ensuring mesh conformity. + */ +class VertexToKnotSpan +{ +public: + /// Set the spatial dimension and number of vertices. + void SetSize(int dimension, int numVertices); + + // The following set and get functions are for a single entry in the array of + // data, for a hanging vertex in the patch topology, with the given 'index'. + // The vertex index is 'v', parent vertices are 'pv', and knot-span is 'ks'. + + /// Set the data for a vertex in 2D. + void SetVertex2D(int index, int v, int ks, + const std::array &pv); + + /// Set the data for a vertex in 3D. + void SetVertex3D(int index, int v, const std::array &ks, + const std::array &pv); + + /// Set the knot-span index for a vertex in 2D. + void SetKnotSpan2D(int index, int ks); + + /// Set the knot-span indices for a vertex in 3D. + void SetKnotSpans3D(int index, const std::array &ks); + + /// Get the data for a vertex in 2D. + void GetVertex2D(int index, int &v, int &ks, + std::array &pv) const; + + /// Get the data for a vertex in 3D. + void GetVertex3D(int index, int &v, std::array &ks, + std::array &pv) const; + + /// Print all the data. + void Print(std::ostream &os) const; + + /// Return the number of vertices. + int Size() const { return data.NumRows(); } + + /// Return the vertex pair representing the parent edge (2D) or face (3D). + std::pair GetVertexParentPair(int index) const; + +private: + int dim; /// Spatial dimension + Array2D data; /// Row-wise data for each vertex. +}; + +/** @brief A class for non-conforming AMR. The class is not used directly by the * user, rather it is an extension of the Mesh class. * * In general, the class is used by MFEM as follows: @@ -348,6 +398,16 @@ public: } } + const VertexToKnotSpan& GetVertexToKnotSpan() const + { + return vertex_to_knotspan; + } + + /// Remap knot-span indices @a vertex_to_knotspan after refinement. + void RefineVertexToKnotSpan(const std::vector> &kvf, + const Array &kvext, + std::map, + std::array> &parentToKV); // coarse/fine transforms @@ -467,7 +527,8 @@ public: /** I/O: Print the mesh in "MFEM NC mesh v1.0" format. If @a comments is non-empty, it will be printed after the first line of the file, and each line should begin with '#'. */ - void Print(std::ostream &out, const std::string &comments = "") const; + void Print(std::ostream &out, const std::string &comments = "", + bool nurbs=false) const; /// I/O: Return true if the mesh was loaded from the legacy v1.1 format. bool IsLegacyLoaded() const { return Legacy; } @@ -483,6 +544,9 @@ public: int PrintMemoryDetail() const; + /// Return true for ParNCMesh with more than one MPI process. + virtual bool IsParallel() const { return false; } + using RefCoord = std::int64_t; static constexpr int MaxElemNodes = @@ -1297,6 +1361,12 @@ protected: /// Load the vertex parent hierarchy from a mesh file. void LoadVertexParents(std::istream &input); + /// Load VertexToKnotSpan data for the NC patch topology mesh of a 2D or 3D + /// MFEM NURBS NC-patch mesh. + void LoadVertexToKnotSpan(std::istream &input); + void LoadVertexToKnotSpan2D(std::istream &input); + void LoadVertexToKnotSpan3D(std::istream &input); + /** Print the "boundary" section of the mesh file. If out == NULL, only return the number of boundary elements. */ int PrintBoundary(std::ostream *out) const; @@ -1339,6 +1409,9 @@ protected: static GeomInfo GI[Geometry::NumGeom]; + /// This is used for a NURBS mesh with this NCMesh as its patch topology. + VertexToKnotSpan vertex_to_knotspan; + #ifdef MFEM_DEBUG public: void DebugLeafOrder(std::ostream &out) const; diff --git a/mesh/ncnurbs.cpp b/mesh/ncnurbs.cpp new file mode 100644 index 0000000000..fd859e6d9a --- /dev/null +++ b/mesh/ncnurbs.cpp @@ -0,0 +1,3866 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#include "ncnurbs.hpp" + +namespace mfem +{ + +using namespace std; + +// Helper functions for NC-NURBS + +void GetShiftedGridPoints2D(int m, int n, int i, int j, int signedShift, + int& sm, int& sn, int& si, int& sj); + +void GetInverseShiftedDimensions2D(int signedShift, int sm, int sn, int &m, + int &n); + +int GetFaceOrientation(const Mesh *mesh, const int face, + const std::array &verts); + +bool Reorder2D(int ori, std::array &s0); + +std::pair QuadrupleToPair(const std::array &q); + +NCNURBSExtension::NCNURBSExtension(std::istream &input, bool spacing) +{ + // Read topology + patchTopo = new Mesh; + patchTopo->LoadNonconformingPatchTopo(input, edge_to_ukv); + nonconformingPT = true; + + Load(input, spacing); +} + +NCNURBSExtension::NCNURBSExtension(const NCNURBSExtension &orig) + : NURBSExtension(orig), + aux_e_meshOffsets(orig.aux_e_meshOffsets), + aux_f_meshOffsets(orig.aux_f_meshOffsets), + aux_e_spaceOffsets(orig.aux_e_spaceOffsets), + aux_f_spaceOffsets(orig.aux_f_spaceOffsets), + auxEdges(orig.auxEdges), + auxFaces(orig.auxFaces), + auxef(orig.auxef) +{ } + +void NCNURBSExtension::GetMasterEdgeEntities( + int edge, Array &edgeV, Array &edgeE, Array &edgeVki) +{ + const int mid = masterEdgeToId.at(edge); + const std::size_t nes = masterEdgeInfo[mid].slaves.size(); + MFEM_ASSERT(masterEdgeInfo[mid].vertices.size() + 1 == nes, ""); + + // Vertices in masterEdgeVerts[mid] are ordered starting + // from the master edge endpoint with lower vertex index. + + Array everts; + patchTopo->GetEdgeVertices(edge, everts); + + edgeV.Append(everts[0]); + edgeVki.Append(0); + + MFEM_ASSERT(masterEdgeInfo[mid].vertices.size() == + masterEdgeInfo[mid].ks.size(), ""); + for (std::size_t i=0; iGetNE(); + + edgeV.Append(everts[1]); + edgeVki.Append(nelem); + + for (std::size_t i=0; i sverts(2); + if (edge_i >= 0) // If a slave edge + { + patchTopo->GetEdgeVertices(edge_i, sverts); + } + else + { + const int auxEdge = -1 - edge_i; + GetAuxEdgeVertices(auxEdge, sverts); + } + + MFEM_ASSERT((sverts[0] == edgeV[i] && + sverts[1] == edgeV[i+1]) || + (sverts[1] == edgeV[i] && + sverts[0] == edgeV[i+1]), ""); + } +} + +void NCNURBSExtension::FindAdditionalFacesSA( + std::map, int> &v2f, + std::set &addParentFaces, + std::vector &facePairs) +{ + for (int f=0; fGetNFaces(); ++f) + { + if (masterFaces.find(f) != masterFaces.end()) + { + continue; // Already a master face + } + + Array edges, ori, verts; + patchTopo->GetFaceEdges(f, edges, ori); + patchTopo->GetFaceVertices(f, verts); + + MFEM_ASSERT(edges.Size() == 4 && verts.Size() == 4, ""); + + const int fn1 = KnotVec(edges[0])->GetNE(); + const int fn2 = KnotVec(edges[1])->GetNE(); + + // Loop over the 2 pairs of opposite sides + for (int p=0; p<2; ++p) // Pair p + { + std::array oppEdges; + const int sideEdge0 = edges[1 - p]; + bool bothMaster = true; + for (int s=0; s<2; ++s) + { + oppEdges[s] = edges[p + 2*s]; + + bool isTrueMasterEdge = false; + if (masterEdges.count(oppEdges[s]) > 0) + { + const int mid = masterEdgeToId.at(oppEdges[s]); + if (masterEdgeInfo[mid].slaves.size() != 0) { isTrueMasterEdge = true; } + } + + if (!isTrueMasterEdge) { bothMaster = false; } + } + + if (!bothMaster) { continue; } + + // Possibly define auxiliary and/or slave faces on this face + + // Check for auxiliary and slave edges + std::vector> sideAuxEdges(2); + std::vector> sideSlaveEdges(2); + for (int s=0; s<2; ++s) + { + const int mid = masterEdgeToId.at(oppEdges[s]); + for (auto edge : masterEdgeInfo[mid].slaves) + { + if (edge < 0) + { + sideAuxEdges[s].Append(-1 - edge); + } + else + { + sideSlaveEdges[s].Append(edge); + } + } + } + + const bool hasAux = sideAuxEdges[0].Size() > 0; + const bool hasSlave = sideSlaveEdges[0].Size() > 0; + + // Find patchTopo vertices in the interior of each side + if (hasAux || hasSlave) + { + std::vector> edgeV(2); + std::vector> edgeE(2); + std::vector> edgeVki(2); + + for (int s=0; s<2; ++s) + { + GetMasterEdgeEntities(oppEdges[s], edgeV[s], edgeE[s], edgeVki[s]); + } + + // Check whether the number and types of edges on opposite + // sides match. If not, skip this pair of sides. + if (edgeE[0].Size() != edgeE[1].Size()) { continue; } + + const int nes = edgeE[0].Size(); + + { + bool matching = true; + for (int i=0; i= 0) != (edgeE[1][i] >= 0)) + { + matching = false; + break; + } + } + + if (!matching) { continue; } + } + + // Check whether edgeV[s] are in the same order or reversed, for + // s=0,1. + bool rev = true; + { + Array sideVerts0; + patchTopo->GetEdgeVertices(sideEdge0, sideVerts0); + sideVerts0.Sort(); + + std::array found{false, false}; + Array ep(2); + for (int e=0; e<2; ++e) // Loop over ends + { + for (int s=0; s<2; ++s) // Loop over sides + { + ep[s] = edgeV[s][e * (edgeV[s].Size() - 1)]; + + for (int i=0; i<2; ++i) + { + if (ep[s] == sideVerts0[i]) + { + found[i] = true; + } + } + } + + ep.Sort(); + if (ep == sideVerts0) + { + rev = false; + } + } + + MFEM_ASSERT(found[0] && found[1], ""); + } + + // Find auxiliary or slave subfaces of face f. + // Note that there may be no master faces in patchTopo->ncmesh. + + for (int i=0; i<2; ++i) + { + MFEM_ASSERT(edgeV[i].Size() == nes + 1, ""); + } + + for (int e=0; e fverts{edgeV[0][e], edgeV[0][e + 1], + edgeV[1][rev ? nes - e - 1 : e + 1], + edgeV[1][rev ? nes - e : e]}; + + // Get indices with respect to the edge. + const int eki = edgeVki[0][e]; + const int eki1 = edgeVki[0][e + 1]; + + const int e1ki = edgeVki[1][rev ? nes - e : e]; + const int e1ki1 = edgeVki[1][rev ? nes - e - 1 : e + 1]; + + // ori_f is the signed shift such that fverts[abs1(ori_f)] is + // closest to vertex 0, verts[0], of parent face f, and the + // relative direction of the ordering is encoded in the sign. + int ori_f = 0; + + // Set the 2D knot-span indices of the 4 vertices in fverts, + // ordered with respect to the face f. + Array2D fki(4,2); + { + // Use eki to get the 2D face knot-span index. + + // Number of elements on the edge. + const int eNE = edgeVki[0][edgeVki[0].Size() - 1]; + + if (p == 0) + { + MFEM_ASSERT(edgeV[0][0] == verts[0] || + edgeV[0][edgeV[0].Size() - 1] == verts[0], + ""); + MFEM_ASSERT(edgeV[1][0] == verts[3] || + edgeV[1][edgeV[0].Size() - 1] == verts[3], + ""); + MFEM_ASSERT(eNE == fn1, ""); + MFEM_ASSERT(edgeVki[1][edgeVki[1].Size() - 1] == fn1, + ""); + + const bool rev0 = edgeV[0][0] != verts[0]; + fki(0,0) = rev0 ? eNE - eki : eki; + fki(0,1) = 0; + + fki(1,0) = rev0 ? eNE - eki1 : eki1; + fki(1,1) = 0; + + if (rev0) { ori_f = -2; } + + // Other side + const bool rev1 = edgeV[1][0] != verts[3]; + + fki(2,0) = rev1 ? eNE - e1ki1 : e1ki1; + fki(2,1) = fn2; + + fki(3,0) = rev1 ? eNE - e1ki : e1ki; + fki(3,1) = fn2; + + MFEM_ASSERT(fki(0,0) == fki(3,0) && + fki(1,0) == fki(2,0), ""); + } + else + { + MFEM_ASSERT(edgeV[0][0] == verts[1] || + edgeV[0][edgeV[0].Size() - 1] == verts[1], + ""); + MFEM_ASSERT(edgeV[1][0] == verts[0] || + edgeV[1][edgeV[0].Size() - 1] == verts[0], + ""); + MFEM_ASSERT(eNE == fn2, ""); + MFEM_ASSERT(edgeVki[1][edgeVki[1].Size() - 1] == fn2, + ""); + + const bool rev0 = edgeV[0][0] != verts[1]; + fki(0,0) = fn1; + fki(0,1) = rev0 ? eNE - eki : eki; + + fki(1,0) = fn1; + fki(1,1) = rev0 ? eNE - eki1 : eki1; + + if (rev0) + { + ori_f = -3; + } + else + { + ori_f = 3; + } + + // Other side + const bool rev1 = edgeV[1][0] != verts[0]; + + fki(2,0) = 0; + fki(2,1) = rev1 ? fn2 - e1ki1 : e1ki1; + + fki(3,0) = 0; + fki(3,1) = rev1 ? fn2 - e1ki : e1ki; + + MFEM_ASSERT(fki(0,1) == fki(3,1) && + fki(1,1) == fki(2,1), ""); + } + } + + // Returns the vertex with minimum knot-span indices. + auto VertexMinKI = [&fki]() + { + int id = -1; + { + std::array kiMin; + for (int j=0; j<2; ++j) + { + kiMin[j] = fki(0,j); + for (int i=1; i<4; ++i) + { + if (fki(i,j) < kiMin[j]) { kiMin[j] = fki(i,j); } + } + } + + for (int i=0; i<4; ++i) + { + if (fki(i,0) == kiMin[0] && fki(i,1) == kiMin[1]) + { + MFEM_ASSERT(id == -1, ""); + id = i; + } + } + } + + MFEM_ASSERT(id >= 0, ""); + return id; + }; + + const std::pair vpair = QuadrupleToPair(fverts); + if (edgeE[0][e] >= 0) + { + const bool vPairTopo = v2f.count(vpair) > 0; + if (!vPairTopo) { continue; } + + const int sface = v2f.at(vpair); + addParentFaces.insert(f); + + // Set facePairs + + // Find the vertex with minimum knot-span indices. + const int vMinID = VertexMinKI(); + + std::array fvertsMasterOrdering; + for (int i=0; i<4; ++i) + { + if (ori_f >= 0) + { + fvertsMasterOrdering[i] = fverts[(vMinID + i) % 4]; + } + else + { + fvertsMasterOrdering[i] = fverts[(vMinID + 4 - i) % 4]; + } + } + + const int ori_sface = GetFaceOrientation(patchTopo, sface, + fvertsMasterOrdering); + + facePairs.emplace_back(FacePairInfo{fverts[vMinID], f, + SlaveFaceInfo{sface, ori_sface, + {fki(vMinID,0), fki(vMinID,1)}, + { + fki((vMinID + 2) % 4,0) - fki(vMinID,0), + fki((vMinID + 2) % 4,1) - fki(vMinID,1) + }}}); + } + else // Auxiliary face + { + const int afid = auxv2f.count(vpair) > 0 ? + auxv2f.at(vpair) : -1; + + addParentFaces.insert(f); + + const int vMinID = VertexMinKI(); + + if (afid >= 0) + { + // Find orientation of ordered vertices for this face, + // in fvertsOrdered, w.r.t. the auxFaces ordering. + std::array fvertsOrdered, afverts; + + int ori_f2 = -1; + for (int i=0; i<4; ++i) + { + afverts[i] = auxFaces[afid].v[i]; + if (ori_f >= 0) + { + fvertsOrdered[i] = fverts[(vMinID + i) % 4]; + } + else + { + fvertsOrdered[i] = fverts[(vMinID + 4 - i) % 4]; + } + + if (fvertsOrdered[i] == afverts[0]) { ori_f2 = i; } + } + + MFEM_ASSERT(ori_f2 >= 0, ""); + + if (fvertsOrdered[(ori_f2 + 1) % 4] != afverts[1]) + { + for (int j=0; j<4; ++j) + { + MFEM_ASSERT(fvertsOrdered[(ori_f2 + 4 - j) % 4] + == afverts[j], ""); + } + + ori_f2 = -1 - ori_f2; + } + else + { + for (int j=0; j<4; ++j) + { + MFEM_ASSERT(fvertsOrdered[(ori_f2 + j) % 4] + == afverts[j], ""); + } + } + + facePairs.emplace_back(FacePairInfo{fverts[vMinID], f, + SlaveFaceInfo{-1 - afid, ori_f2, + {fki(vMinID,0), fki(vMinID,1)}, + { + fki((vMinID + 2) % 4,0) - fki(vMinID,0), + fki((vMinID + 2) % 4,1) - fki(vMinID,1) + }}}); + } + else + { + // Create a new auxiliary face. + const int auxFaceId = auxFaces.size(); + // Find the knot-span indices of the vertices in fverts, + // with respect to the parent face. + AuxiliaryFace auxFace; + for (int i=0; i<4; ++i) + { + if (ori_f >= 0) + { + auxFace.v[i] = fverts[(vMinID + i) % 4]; + } + else + { + auxFace.v[i] = fverts[(vMinID + 4 - i) % 4]; + } + } + + // Orientation is defined as 0 for a new auxiliary face. + ori_f = 0; + + auxFace.parent = f; + auxFace.ori = ori_f; + for (int i=0; i<2; ++i) + { + auxFace.ksi0[i] = fki(vMinID,i); + auxFace.ksi1[i] = fki((vMinID + 2) % 4,i); + } + + auxv2f[vpair] = auxFaces.size(); + auxFaces.push_back(auxFace); + + facePairs.emplace_back(FacePairInfo{fverts[vMinID], f, + SlaveFaceInfo{-1 - auxFaceId, ori_f, + {fki(vMinID,0), fki(vMinID,1)}, + { + fki((vMinID + 2) % 4,0) - fki(vMinID,0), + fki((vMinID + 2) % 4,1) - fki(vMinID,1) + }}}); + } + } + } + } + } // Pair (p) loop + } // f +} + +void NCNURBSExtension::ProcessFacePairs(int start, int midStart, + const std::vector> &parentSize, + std::vector &parentVerts, + const std::vector &facePairs) +{ + const int nfpairs = facePairs.size(); + const bool is3D = Dimension() == 3; + MFEM_VERIFY(nfpairs > 0 || !is3D, ""); + int midPrev = -1; + int orientation = 0; + for (int q=start; q pv; + for (int k=0; k<4; ++k) { pv[k] = parentVerts[(4*mid) + k]; } + const int ori = GetFaceOrientation(patchTopo, parentFace, pv); + // Ori is the signed shift such that pv[abs1(ori)] is vertex 0 of + // parentFace, and the relative direction of the ordering is encoded in + // the sign. + if (q > start && midPrev >= 0) + { + // For the previous parentFace, use previous orientation to reorder + // masterFaceSlaves, masterFaceSlaveCorners, masterFaceSizes. + std::array s0; + masterFaceInfo[midPrev].rev = Reorder2D(orientation, s0); + masterFaceInfo[midPrev].s0 = s0; + } + + orientation = ori; + midPrev = mid; + } // next parent face + + slaveFaces.emplace_back(SlaveFaceInfo{childFace, cpori, {i, j}, + {nfe1, nfe2}}); + + const int si = slaveFaces.size() - 1; + masterFaceInfo[mid].slaves.push_back(si); + masterFaceInfo[mid].slaveCorners.push_back(v0); + masterFaceInfo[mid].ne[0] = parentSize[mid][0]; + masterFaceInfo[mid].ne[1] = parentSize[mid][1]; + } // Loop (q) over facePairs + + if (midPrev >= 0) + { + std::array s0; + masterFaceInfo[midPrev].rev = Reorder2D(orientation, s0); + masterFaceInfo[midPrev].s0 = s0; + } +} + +void NCNURBSExtension::GetAuxEdgeVertices(int auxEdge, Array &verts) const +{ + verts.SetSize(2); + for (int i=0; i<2; ++i) { verts[i] = auxEdges[auxEdge].v[i]; } +} + +void NCNURBSExtension::GetAuxFaceVertices(int auxFace, Array &verts) const +{ + verts.SetSize(4); + for (int i=0; i<4; ++i) { verts[i] = auxFaces[auxFace].v[i]; } +} + +void NCNURBSExtension::GetAuxFaceEdges(int auxFace, Array &edges) const +{ + edges.SetSize(4); + Array verts(2); + for (int i=0; i<4; ++i) + { + for (int j=0; j<2; ++j) { verts[j] = auxFaces[auxFace].v[(i + j) % 4]; } + + verts.Sort(); + const std::pair edge_v(verts[0], verts[1]); + // Note that v2e is a map only for conforming patchTopo->ncmesh edges. + // Auxiliary edges are in auxv2e, not in v2e. + if (v2e.count(edge_v) > 0) + { + edges[i] = v2e.at(edge_v); // patchTopo edge + } + else // Auxiliary edge + { + edges[i] = -1 - auxv2e.at(edge_v); + } + } +} + +// Negative indices are for array `b`. Nonnegative indices are for array `a`, +// except for index `a.Size()`, corresponding to `b[0]`. +int OffsetHelper(int i, int j, const Array &a, const Array &b) +{ + if (i < 0) + { + return b[-1 - i + j]; + } + else if (i + j < a.Size()) + { + return a[i + j]; + } + else + { + return b[0]; + } +} + +int NCNURBSExtension::GetEdgeOffset(bool dof, int edge, int increment) const +{ + return OffsetHelper(edge, increment, dof ? e_spaceOffsets : e_meshOffsets, + dof ? aux_e_spaceOffsets : aux_e_meshOffsets); +} + +int NCNURBSExtension::GetFaceOffset(bool dof, int face, int increment) const +{ + return OffsetHelper(face, increment, dof ? f_spaceOffsets : f_meshOffsets, + dof ? aux_f_spaceOffsets : aux_f_meshOffsets); +} + +void NCNURBSExtension::GetMasterEdgeDofs(bool dof, int me, + Array &dofs) const +{ + MFEM_ASSERT(masterEdges.count(me) > 0, "Not a master edge"); + const int mid = masterEdgeToId.at(me); + + MFEM_ASSERT(masterEdgeInfo[mid].vertices.size() == + masterEdgeInfo[mid].slaves.size() - 1, ""); + + const Array& v_offsets = dof ? v_spaceOffsets : v_meshOffsets; + const std::size_t nes = masterEdgeInfo[mid].slaves.size(); + for (std::size_t s=0; s svert; + if (slaveId >= 0) + { + patchTopo->GetEdgeVertices(slaveId, svert); + } + else // Auxiliary edge + { + GetAuxEdgeVertices(-1 - slaveId, svert); + } + + bool reverse = false; + if (nes > 1) + { + const int mev = masterEdgeInfo[mid].vertices[std::max((int) s - 1,0)]; + MFEM_ASSERT(mev == svert[0] || mev == svert[1], ""); + if (s == 0) + { + // In this case, mev is the second vertex of the edge. + if (svert[0] == mev) { reverse = true; } + } + else + { + // In this case, mev is the first vertex of the edge. + if (svert[1] == mev) { reverse = true; } + } + } + + const int eos = GetEdgeOffset(dof, slaveId, 0); + const int eos1 = GetEdgeOffset(dof, slaveId, 1); + const int nvs = eos1 - eos; + MFEM_ASSERT(nvs >= 0, ""); + + // Add all slave edge vertices/DOFs + + Array sdofs(nvs); + for (int j=0; jIsMasterEdge(edges[i]); + edgeMasterOffset[i] = mos; + + if (edgeMaster[i]) + { + Array mdof; + Ext->GetMasterEdgeDofs(dof, edges[i], mdof); + masterDofs.Append(mdof); + mos += mdof.Size(); + } + } +} + +void NCNURBSExtension::GetFaceOrdering(int sf, int n1, int n2, int v0, + int e1, int e2, Array &perm) const +{ + perm.SetSize(n1 * n2); + + // The ordering of entities in the face is based on the vertices. + + Array faceEdges, ori, evert, e2vert, vert; + patchTopo->GetFaceEdges(sf, faceEdges, ori); + patchTopo->GetFaceVertices(sf, vert); + patchTopo->GetEdgeVertices(faceEdges[e1], evert); + MFEM_ASSERT(evert[0] == v0 || evert[1] == v0, ""); + + bool d[2]; + d[0] = (evert[0] == v0); + + const int v10 = d[0] ? evert[1] : evert[0]; + + // The face has {fn1,fn2} interior entities, with ordering based on `vert`. + // Now we find these sizes by first finding the edge with vertices [v0, v10]. + int e0 = -1; + for (int i=0; i<4; ++i) + { + patchTopo->GetEdgeVertices(faceEdges[i], evert); + if ((evert[0] == v0 && evert[1] == v10) || + (evert[1] == v0 && evert[0] == v10)) { e0 = i; } + } + + MFEM_ASSERT(e0 >= 0, ""); + + const bool tr = e0 % 2 == 1; // True means (fn1,fn2) == (n2,n1) + + patchTopo->GetEdgeVertices(faceEdges[e2], evert); + MFEM_ASSERT(evert[0] == v10 || evert[1] == v10, ""); + d[1] = (evert[0] == v10); + + const int v11 = d[1] ? evert[1] : evert[0]; + + int v01 = -1; + for (int i=0; i<4; ++i) + { + if (vert[i] != v0 && vert[i] != v10 && vert[i] != v11) { v01 = vert[i]; } + } + + MFEM_ASSERT(v01 >= 0 && v01 == vert.Sum() - v0 - v10 - v11, ""); + + // Translate indices [v0, v10, v11, v01] to pairs of indices in {0,1}. + constexpr char ipair[4][2] = {{0, 0}, {1, 0}, {1, 1}, {0, 1}}; + int f00[2]; + + int allv[4] = {v0, v10, v11, v01}; + int locv[4]; + for (int i=0; i<4; ++i) + { + locv[i] = -1; + for (int j=0; j<4; ++j) + { + if (vert[j] == allv[i]) + { + locv[i] = j; + } + } + + MFEM_ASSERT(locv[i] >= 0, ""); + } + + for (int i=0; i<2; ++i) { f00[i] = ipair[locv[0]][i]; } + + const int i0 = f00[0]; + const int j0 = f00[1]; + + for (int i=0; i &a, + Array2D &b) +{ + const int m = a.NumRows(); + const int n = a.NumCols(); + + b.SetSize(m, n); + + const int s0 = i0 == 0 ? 1 : -1; + const int s1 = j0 == 0 ? 1 : -1; + for (int i=0; i &perm) +{ + const int oriAbs = ori < 0 ? -1 - ori : ori; + + for (int i=0; i<4; ++i) + { + if (ori < 0) + { + perm[i] = (oriAbs - i + 4) % 4; + } + else + { + perm[i] = (ori + i) % 4; + } + } +} + +// Append master face DOFs to masterDofs. +void NURBSPatchMap::SetMasterFaces(bool dof) +{ + faceMaster.SetSize(faces.Size()); + faceMasterOffset.SetSize(faces.Size()); + + // The loop over master edges is already done by SetMasterEdges, and now we + // append face DOFs to masterDofs. + + int mos = masterDofs.Size(); + for (int i=0; iIsMasterFace(faces[i]); + faceMasterOffset[i] = mos; + + if (!faceMaster[i]) { continue; } + + Array2D mdof; + Ext->GetMasterFaceDofs(dof, faces[i], mdof); + if (mdof.NumRows() == 0) + { + faceMaster[i] = false; + continue; + } + + for (int j=0; j &dofs) const +{ + const int mid = masterFaceToId.at(mf); + const bool rev = masterFaceInfo[mid].rev; + const int s0i = masterFaceInfo[mid].s0[0]; + const int s0j = masterFaceInfo[mid].s0[1]; + const int n1orig = masterFaceInfo[mid].ne[0]; + const int n2orig = masterFaceInfo[mid].ne[1]; + + // Skip master faces with no slave faces (only having slave edges). + if (n1orig == 0 && n2orig == 0) { return; } + + const int n1 = rev ? n2orig : n1orig; + const int n2 = rev ? n1orig : n2orig; + + MFEM_ASSERT((n1 > 1 || n2 > 1) && + n1 * n2 >= (int) masterFaceInfo[mid].slaves.size(), + "Inconsistent number of faces"); + + int fcnt = 0; + for (auto slaveId : masterFaceInfo[mid].slaves) + { + fcnt += slaveFaces[slaveId].ne[0] * slaveFaces[slaveId].ne[1]; + } + + MFEM_VERIFY(fcnt == n1 * n2, ""); + MFEM_VERIFY((int) masterFaceInfo[mid].slaveCorners.size() <= n1 * n2, ""); + + // Set an array of vertices or DOFs for the interior of this master face. + // Set master face entity dimensions. + int mnf1, mnf2; + Array medges; + { + Array mori; + patchTopo->GetFaceEdges(mf, medges, mori); + } + + MFEM_ASSERT(medges.Size() == 4, ""); + + if (dof) + { + mnf1 = KnotVec(medges[0])->GetNCP() - 2; + mnf2 = KnotVec(medges[1])->GetNCP() - 2; + } + else + { + mnf1 = KnotVec(medges[0])->GetNE() - 1; + mnf2 = KnotVec(medges[1])->GetNE() - 1; + } + + // Set dimensions for a single mesh edge. + const int sne1 = (mnf1 - n1 + 1) / n1; + const int sne2 = (mnf2 - n2 + 1) / n2; + + MFEM_ASSERT(sne1 * n1 == mnf1 - n1 + 1, ""); + MFEM_ASSERT(sne2 * n2 == mnf2 - n2 + 1, ""); + + const Array &v_offsets = dof ? v_spaceOffsets : v_meshOffsets; + + Array2D mdof(mnf1, mnf2); + mdof = -1; + + bool consistent = true; + + for (std::size_t s=0; s orderedVertices; + std::array edgeBdry; + std::set vbdry; + int nf1 = 0, nf2 = 0; + auto SetEdgeEntries = [&](int eidx, int edge, const Array &evert, + int &vstart) + { + const bool reverse = (vstart == evert[1]); + const int vend = evert.Sum() - vstart; + vstart = vend; + + // Skip edges on the boundary of the master face. + if (edgeBdry[eidx]) + { + for (auto v : evert) { vbdry.insert(v); } + return; + } + const bool horizontal = (eidx % 2 == 0); + const int nf_e = horizontal ? nf1 : nf2; + + // Edge entities + const int eos = GetEdgeOffset(dof, edge, 0); +#ifdef MFEM_DEBUG + const int eos1 = GetEdgeOffset(dof, edge, 1); +#endif + const bool edgeIsMaster = masterEdges.count(edge) > 0; + + Array edofs; + if (edgeIsMaster) + { + // This edge is a slave edge and a master edge. Instead of + // getting DOFs from e_offsets, take them from the slave edges + // of this edge. + GetMasterEdgeDofs(dof, edge, edofs); + } + else + { + MFEM_ASSERT(eos1 - eos == nf_e, ""); + + edofs.SetSize(nf_e); + for (int j=0; j= 0, then vertex ori of the aux face is closest to vertex 0 + // of the parent face. If ori < 0, then the orientations of the aux face + // and parent face are reversed. + + // NOTE: When an aux face is first defined, it is constructed with + // orientation 0 w.r.t. its parent face. However, it can be part of + // another parent face. In this case, the original aux face index is + // used, but it is paired with a different parent face, with + // possibly different orientation w.r.t. that parent face. In + // NURBSPatchMap::SetMasterFaces, the DOFs are set on the parent + // face in its knot-span indices, with an aux face of possibly nonzero + // orientation. When the orientation is nonzero, the aux face DOFs + // must be reordered for the parent face, based on orientation. + + int onf1, onf2; + GetInverseShiftedDimensions2D(ori, nf1, nf2, onf1, onf2); + for (int k=0; k faceEdges; + GetAuxFaceEdges(auxFace, faceEdges); + + std::array perm; + GetVertexOrdering(ori, perm); + + int vstart = v0; + for (int eidx=0; eidx<4; ++eidx) + { + orderedVertices[eidx] = vstart; + MFEM_ASSERT(orderedVertices[eidx] == + auxFaces[auxFace].v[perm[eidx]], ""); + const int eperm = ori < 0 ? (perm[eidx] - 1 + 4) % 4 : perm[eidx]; + const int edge = faceEdges[eperm]; + Array evert; + if (edge >= 0) + { + patchTopo->GetEdgeVertices(edge, evert); + } + else + { + const int auxEdge = -1 - edge; + GetAuxEdgeVertices(auxEdge, evert); + } + MFEM_ASSERT(evert[0] == vstart || evert[1] == vstart, ""); + SetEdgeEntries(eidx, edge, evert, vstart); + } // eidx + } + else // slaveId >= 0 + { + // Determine which slave face edges are in the first and second + // dimensions of the master face, by using ori. + int e1 = -1, e2 = -1; + { + const int aori = ori < 0 ? -1 - ori : ori; + if (aori % 2 == 0) + { + e1 = 0; + e2 = 1; + } + else + { + e1 = 1; + e2 = 0; + } + + if (ori < 0) + { + // Swap e1, e2 + const int sw = e1; + e1 = e2; + e2 = sw; + } + } + + // Now, e1 is one of the two horizontal edges in the master face + // directions. If it does not touch v0, then take the other horizontal + // edge. Do the same for e2. + + Array sedges; + { + Array sori; + patchTopo->GetFaceEdges(slaveId, sedges, sori); + } + + int v1 = -1; + { + Array evert; + patchTopo->GetEdgeVertices(sedges[e1], evert); + if (evert.Find(v0) == -1) + { + e1 += 2; + patchTopo->GetEdgeVertices(sedges[e1], evert); + } + + const int idv0 = evert.Find(v0); + MFEM_ASSERT(idv0 >= 0, ""); + v1 = evert[1 - idv0]; + + patchTopo->GetEdgeVertices(sedges[e2], evert); + if (evert.Find(v1) == -1) + { + e2 += 2; + patchTopo->GetEdgeVertices(sedges[e2], evert); + } + + MFEM_ASSERT(evert.Find(v1) >= 0, ""); + } + + // Set slave face entity dimensions. + if (dof) + { + nf1 = KnotVec(sedges[e1])->GetNCP() - 2; + nf2 = KnotVec(sedges[e2])->GetNCP() - 2; + } + else + { + nf1 = KnotVec(sedges[e1])->GetNE() - 1; + nf2 = KnotVec(sedges[e2])->GetNE() - 1; + } + + MFEM_ASSERT(sne1 * ne1 == nf1 - ne1 + 1, ""); + MFEM_ASSERT(sne2 * ne2 == nf2 - ne2 + 1, ""); + + MFEM_ASSERT(nvs == nf1 * nf2, ""); + + // Find the DOFs of the slave face ordered for the master face. We know + // that e1 and e2 are the local indices of the slave face edges on the + // bottom and right side, with respect to the master face directions. + Array perm; + GetFaceOrdering(slaveId, nf1, nf2, v0, e1, e2, perm); + + for (int k=0; k edgeOrder{e1, e2, (e1 + 2) % 4, (e2 + 2) % 4}; + + // Horizontal edges + edgeBdry[0] = sJ == 0; + edgeBdry[2] = sJ + ne2 == n2; + + // Vertical edges + edgeBdry[1] = sI + ne1 == n1; + edgeBdry[3] = sI == 0; + + int vstart = v0; + for (int eidx=0; eidx<4; ++eidx) + { + orderedVertices[eidx] = vstart; + const int edge = sedges[edgeOrder[eidx]]; + Array evert; + patchTopo->GetEdgeVertices(edge, evert); + SetEdgeEntries(eidx, edge, evert, vstart); + } // eidx + } + + // Set entries at vertices of this face, if interior to the master face. + for (int vidx=0; vidx<4; ++vidx) + { + const int v = orderedVertices[vidx]; + if (vbdry.count(v) == 0) // If not on the master face boundary. + { + int m1, m2; + if (vidx == 0) + { + m1 = os1 - 1; + m2 = os2 - 1; + } + else if (vidx == 1) + { + m1 = os1 + nf1; + m2 = os2 - 1; + } + else if (vidx == 2) + { + m1 = os1 + nf1; + m2 = os2 + nf2; + } + else + { + m1 = os1 - 1; + m2 = os2 + nf2; + } + + if (!ConsistentlySetEntry(v_offsets[v], mdof(m1, m2))) + { + consistent = false; + } + } + } // vidx + } // Loop (s) over slave faces. + + // Let `ori` be the signed shift such that pv[abs1(ori)] is vertex 0 of + // parentFace, and the relative direction of the ordering is encoded in the + // sign. Here, pv means parent vertices, as ordered in the mesh file. Then + // Reorder2D takes `ori` and computes (s0i, s0j) as the corresponding integer + // coordinates in {0,1}x{0,1}. Thus reference vertex (s0i, s0j) of pv (parent + // vertices in mesh file) is vertex (0, 0) of parentFace. Currently, mdof is + // in the ordering of pv, and the entries are now reordered, according to + // parentFace vertex ordering, for appending to masterDofs. That means the + // first entry appended to masterDofs should be the entry of mdof + // corresponding to (s0i, s0j). + + ReorderArray2D(s0i, s0j, mdof, dofs); + const bool all_set = dofs.NumRows() * dofs.NumCols() == 0 || dofs.Min() >= 0; + MFEM_VERIFY(all_set && consistent, ""); +} + +int NURBSPatchMap::GetMasterEdgeDof(const int e, const int i) const +{ + const int os = edgeMasterOffset[e]; + return masterDofs[os + i]; +} + +int NURBSPatchMap::GetMasterFaceDof(const int f, const int i) const +{ + const int os = faceMasterOffset[f]; + return masterDofs[os + i]; +} + +void NCNURBSExtension::ProcessVertexToKnot2D(const VertexToKnotSpan &v2k, + std::set &reversedParents, + std::vector &edgePairs) +{ + auxEdges.clear(); + auxv2e.clear(); + + const int nv2k = v2k.Size(); + + int prevParent = -1; + int prevV = -1; + int prevKI = -1; + for (int i=0; i pv; + v2k.GetVertex2D(i, tv, ks, pv); + + // Given that the parent Mesh is not yet constructed, and all we have at + // this point is patchTopo->ncmesh, we should only define master/slave + // edges by indices in patchTopo->ncmesh, as done in the case of nonempty + // nce.masters. Now find the edge in patchTopo->ncmesh with vertices + // (pv[0], pv[1]), and define it as a master edge. + + const std::pair parentPair(pv[0] < pv[1] ? pv[0] : pv[1], + pv[0] < pv[1] ? pv[1] : pv[0]); + + MFEM_ASSERT(v2e.count(parentPair) > 0, "Vertex pair not found"); + const int parentEdge = v2e[parentPair]; + masterEdges.insert(parentEdge); + + const int kv = KnotInd(parentEdge); + parentToKV[parentPair] = std::array {kv, -1}; + + const bool rev = pv[1] < pv[0]; + if (rev) { reversedParents.insert(parentEdge); } + + // Note that the logic here assumes that the "vertex_to_knotspan" data in + // the mesh file has vertices in order of ascending knotIndex. + + const bool newParentEdge = (prevParent != parentEdge); + const int v0 = newParentEdge ? pv[0] : prevV; + + if (ks == 1) { MFEM_ASSERT(newParentEdge, ""); } + + // Find the edge in patchTopo->ncmesh with vertices (v0, tv), and define + // it as a slave edge. + + const std::pair childPair(v0 < tv ? v0 : tv, v0 < tv ? tv : v0); + const bool childPairTopo = v2e.count(childPair) > 0; + if (!childPairTopo) + { + // Check whether childPair is in auxEdges. + if (auxv2e.count(childPair) == 0) + { + // Create a new auxiliary edge + auxv2e[childPair] = auxEdges.size(); + auxEdges.emplace_back(AuxiliaryEdge{pv[0] < pv[1] ? + parentEdge : -1 - parentEdge, + {childPair.first, childPair.second}, + {newParentEdge ? 0 : prevKI, ks}}); + } + } + + const int childEdge = childPairTopo ? v2e[childPair] : -1 - auxv2e[childPair]; + + // Check whether this is the final vertex in this parent edge. Note that + // the logic for comparing (pv[0],pv[1]) to the next parents assumes the + // ordering does not change, which is ensured by the assumption that the + // knot-span index is increasing. + bool finalVertex = (i == nv2k-1); + if (i < nv2k-1) + { + int tv_next, ks_next; + std::array pv_next; + v2k.GetVertex2D(i + 1, tv_next, ks_next, pv_next); + if (pv_next[0] != pv[0] || pv_next[1] != pv[1]) + { + finalVertex = true; + } + } + + edgePairs.emplace_back(tv, ks, childEdge, parentEdge); + + if (finalVertex) + { + // Also find the edge with vertices (tv, pv[1]), and define it as a + // slave edge. + const std::pair finalChildPair(tv < pv[1] ? tv : pv[1], + tv < pv[1] ? pv[1] : tv); + const bool finalChildPairTopo = v2e.count(finalChildPair) > 0; + if (!finalChildPairTopo) + { + // Check whether finalChildPair is in auxEdges. + if (auxv2e.count(finalChildPair) == 0) + { + // Create a new auxiliary edge + auxv2e[finalChildPair] = auxEdges.size(); + + // -1 denotes `ne` at endpoint + auxEdges.emplace_back(AuxiliaryEdge{pv[0] < pv[1] ? + -1 - parentEdge : parentEdge, + {finalChildPair.first, finalChildPair.second}, + {ks, -1}}); + } + } + + const int finalChildEdge = finalChildPairTopo ? v2e[finalChildPair] : + -1 - auxv2e[finalChildPair]; + edgePairs.emplace_back(-1, -1, finalChildEdge, parentEdge); + } + + prevV = tv; + prevKI = ks; + prevParent = parentEdge; + } // loop over vertices in vertex_to_knotspan +} + +void NCNURBSExtension::ProcessVertexToKnot3D( + const VertexToKnotSpan &v2k, + const std::map, int> &v2f, + std::vector> &parentSize, + std::vector &edgePairs, + std::vector &facePairs, + std::vector &parentFaces, + std::vector &parentVerts) +{ + auxEdges.clear(); + auxFaces.clear(); + auxv2e.clear(); + auxv2f.clear(); + + const int nv2k = v2k.Size(); + + // Note that the logic here assumes that the "vertex_to_knotspan" data in the + // mesh file has vertices in order of ascending (k1,k2), with k2 being the + // fast variable, and with corners skipped. + + // Find parentOffset, which stores the indices in v2k at which parent faces + // start. + int prevParent = -1; + std::vector parentOffset; + std::vector parentV2Kedge; + int n1 = 0; + int n2 = 0; + int n1min = 0; + int n2min = 0; + for (int i = 0; i < nv2k; ++i) + { + int tv; + std::array ks; + std::array pv; + v2k.GetVertex3D(i, tv, ks, pv); + + // The face with vertices (pv[0], pv[1], pv[2], pv[3]) is defined as a + // parent face. + const std::pair parentPair = QuadrupleToPair(pv); + const int parentFace = v2f.at(parentPair); + const bool newParentFace = (prevParent != parentFace); + if (newParentFace) + { + parentOffset.push_back(i); + parentFaces.push_back(parentFace); + + // Find the knotvectors for the first two edges this face. + { + Array edges, ori, verts; + patchTopo->GetFaceEdges(parentFace, edges, ori); + patchTopo->GetFaceVertices(parentFace, verts); + + std::array kv = {-1, -1}; + for (int e=0; e<2; ++e) + { + // Find the edge with vertices pv[e] and pv[e+1]. + for (auto edge : edges) + { + Array evert; + patchTopo->GetEdgeVertices(edge, evert); + const bool matching = (evert[0] == pv[e] && evert[1] == pv[e+1]) || + (evert[1] == pv[e] && evert[0] == pv[e+1]); + if (matching) { kv[e] = KnotInd(edge); } + } + + MFEM_ASSERT(kv[e] >= 0, ""); + } + + parentToKV[parentPair] = std::array {kv[0], kv[1]}; + } + + if (i > 0) + { + // In the case of only 1 element in the 1-direction, it is assumed + // that the 2-direction has more than 1 element, so there are + // knot-spans (0, ki2) and (1, ki2) for 0 < ki2 < n2. This will + // result in n1 = 0, which should be 1. Also, n2 will be 1 less than + // it should be. Similarly for the situation with directions + // reversed. + const int n1range = n1 - n1min; + const int n2range = n2 - n2min; + parentV2Kedge.push_back(n1range == 0 || n2range == 0); + } + + auto getEdgeNE = [&](int d) + { + Array ev(2); + for (int j=0; j<2; ++j) { ev[j] = pv[j + d]; } + ev.Sort(); + return KnotVecNE(v2e.at(std::pair(ev[0], ev[1]))); + }; + parentSize.emplace_back(std::array {getEdgeNE(0), getEdgeNE(1)}); + + n1 = ks[0]; // Finding max of ks[0] + n2 = ks[1]; // Finding max of ks[1] + + n1min = n1; + n2min = n2; + } + else + { + n1 = std::max(n1, ks[0]); // Finding max of ks[0] + n2 = std::max(n2, ks[1]); // Finding max of ks[1] + + n1min = std::min(n1min, ks[0]); + n2min = std::min(n2min, ks[1]); + } + + prevParent = parentFace; + } + + { + const int n1range = n1 - n1min; + const int n2range = n2 - n2min; + parentV2Kedge.push_back(n1range == 0 || n2range == 0); + } + + const int numParents = parentOffset.size(); + parentOffset.push_back(nv2k); + + std::set visitedParentEdges; + std::map edgePairOS; + bool consistent = true; + + for (int parent = 0; parent < numParents; ++parent) + { + const int parentFace = parentFaces[parent]; + + int parentEdges[4]; + bool parentEdgeRev[4]; + + int tvi; + std::array ks; + std::array pv; + v2k.GetVertex3D(parentOffset[parent], tvi, ks, pv); + + // Set all 4 edges of the parent face as master edges. + { + Array ev(2); + for (int i=0; i<4; ++i) + { + for (int j=0; j<2; ++j) + { + ev[j] = pv[(i + j) % 4]; + } + + const bool reverse = (ev[1] < ev[0]); + parentEdgeRev[i] = reverse; + + ev.Sort(); + + const std::pair edge_i(ev[0], ev[1]); + + const int parentEdge = v2e.at(edge_i); + masterEdges.insert(parentEdge); + parentEdges[i] = parentEdge; + } + } + + n1 = parentSize[parent][0]; + n2 = parentSize[parent][1]; + Array2D gridVertex(n1 + 1, n2 + 1); + gridVertex = -1; + + gridVertex(0,0) = pv[0]; + gridVertex(n1,0) = pv[1]; + gridVertex(n1,n2) = pv[2]; + gridVertex(0,n2) = pv[3]; + + for (int i=0; i<4; ++i) { parentVerts.push_back(pv[i]); } + + int r1min = -1; + int r1max = -1; + int r2min = -1; + int r2max = -1; + + for (int i = parentOffset[parent]; i < parentOffset[parent + 1]; ++i) + { + v2k.GetVertex3D(i, tvi, ks, pv); + gridVertex(ks[0], ks[1]) = tvi; + if (i == parentOffset[parent]) + { + // Initialize min/max + r1min = ks[0]; + r1max = ks[0]; + + r2min = ks[1]; + r2max = ks[1]; + } + else + { + r1min = std::min(r1min, ks[0]); + r1max = std::max(r1max, ks[0]); + + r2min = std::min(r2min, ks[1]); + r2max = std::max(r2max, ks[1]); + } + } // loop over vertices in v2k + + MFEM_ASSERT((r1max - r1min + 1) * (r2max - r2min + 1) >= + parentOffset[parent + 1] - parentOffset[parent], ""); + + std::array kvi; + if (kvf.size() > 0) + { + const std::pair parentPair = v2k.GetVertexParentPair( + parentOffset[parent]); + std::array kv = parentToKV.at(parentPair); + for (int i=0; i<2; ++i) { kvi[i] = kv[i]; } + } + + // Default refinement factor + const int rf = ref_factors.Size() == 3 ? ref_factors[0] : 1; + + int n1orig = kvf_coarse.size() > 0 ? kvf_coarse[kvi[0]].Size() : n1 / rf; + int n2orig = kvf_coarse.size() > 0 ? kvf_coarse[kvi[1]].Size() : n2 / rf; + + if (kvf.size() > 0 && kvf_coarse.size() == 0) + { + n1orig = kvf[kvi[0]].Size(); + n2orig = kvf[kvi[1]].Size(); + } + + if (kvf.size() > 0) + { + MFEM_ASSERT(kvf[kvi[0]].Sum() == n1 && kvf[kvi[1]].Sum() == n2, ""); + } + + std::vector> cgrid(2); + std::array n_orig = {n1orig, n2orig}; + for (int dir=0; dir<2; ++dir) + { + cgrid[dir].SetSize(n_orig[dir] + 1); + cgrid[dir][0] = 0; + for (int ii = 0; ii < n_orig[dir]; ++ii) + { + const int iir = parentEdgeRev[dir] ? n_orig[dir] - 1 - ii : ii; + + int d = 1; // refinement factor + + if (kvf_coarse.size() > 0) + { + d = kvf_coarse[kvi[dir]][iir]; + } + else if (kvf.size() > 0) + { + d = kvf[kvi[dir]][iir]; + } + + cgrid[dir][ii + 1] = cgrid[dir][ii] + d; + } + } + + MFEM_ASSERT(cgrid[0][n_orig[0]] == n1 && cgrid[1][n_orig[1]] == n2, ""); + + bool allset = true; + bool hasSlaveFaces = false; + bool hasAuxFace = false; + + for (int ii=0; ii<=n_orig[0]; ++ii) + { + const int i = cgrid[0][ii]; + for (int jj=0; jj<=n_orig[1]; ++jj) + { + const int j = cgrid[1][jj]; + if (gridVertex(i,j) < 0) + { + allset = false; + } + else if (0 < i && i < n1 && 0 < j && j < n2) + { + hasSlaveFaces = true; + } + } + } + + auto SetFacePairOnGridRange = [&](int i0, int i1, int j0, int j1) + { + std::array cv{gridVertex(i0, j0), gridVertex(i1, j0), + gridVertex(i1, j1), gridVertex(i0, j1)}; + const std::pair childPair = QuadrupleToPair(cv); + + // min(cv) may be negative, if gridVertex is not set everywhere. + if (childPair.first < 0) { return; } + + const int d0 = i1 - i0; + const int d1 = j1 - j0; + + const bool childPairTopo = v2f.count(childPair) > 0; + if (childPairTopo) + { + const int childFace = v2f.at(childPair); + const int ori = GetFaceOrientation(patchTopo, childFace, cv); + // ori gives the orientation and index of cv matching the first + // vertex of childFace. + facePairs.emplace_back( + FacePairInfo{cv[0], parentFace, SlaveFaceInfo{childFace, + ori, {i0, j0}, {d0, d1}}}); + } + else + { + // Check whether the parent face is on the boundary. + const Mesh::FaceInformation faceInfo = patchTopo->GetFaceInformation( + parentFace); + const bool bdryParentFace = faceInfo.IsBoundary(); + + if (!allset && !bdryParentFace) + { + hasAuxFace = true; + // Check whether childPair is in auxFaces. + if (auxv2f.count(childPair) == 0) + { + // Create a new auxiliary face + auxv2f[childPair] = auxFaces.size(); + AuxiliaryFace auxFace; + for (int k=0; k<4; ++k) { auxFace.v[k] = cv[k]; } + + auxFace.parent = parentFace; + // Orientation is defined as 0 for a new auxiliary face. + auxFace.ori = 0; + auxFace.ksi0[0] = i0; + auxFace.ksi0[1] = j0; + auxFace.ksi1[0] = i1; + auxFace.ksi1[1] = j1; + + auxFaces.push_back(auxFace); + facePairs.emplace_back( + FacePairInfo{cv[0], parentFace, + SlaveFaceInfo{-1 - auxv2f[childPair], + 0, {i0, j0}, {d0, d1}}}); + } + } + } + }; + + // Loop over child faces and set facePairs, as well as auxiliary faces. + for (int ii=0; ii 0; + + if (!parentVisited) + { + edgePairOS[parentEdge] = edgePairs.size(); + edgePairs.resize(edgePairs.size() + ne); + } + + int tvprev = -1; + int kiprev = -1; + bool lagTV = false; + int firstEdge = -1; + int os_e = 0; + + // Loop edges in direction `dir` + for (int e_orig = 0; e_orig < n_orig[dir-1]; ++e_orig) + { + const int e_i = os_e; + const int e_orig_rev = reverse ? n_orig[dir-1] - 1 - e_orig : e_orig; + int de = rf; + if (kvf_coarse.size() > 0) + { + de = kvf_coarse[kvi[dir - 1]][e_orig_rev]; + } + else if (kvf.size() > 0) + { + de = kvf[kvi[dir - 1]][e_orig_rev]; + } + + os_e += de; + + // For both directions, side s=0 has increasing indices and s=1 + // has decreasing indices. + + const int i0 = e_i; + const int i1 = e_i + de; + + // Edge index with respect to the master edge. + const int e_idx = reverse ? ne - e_i - de : e_i; + + Array cv(2); + if (dir == 1) + { + cv[0] = gridVertex(i0,s*n2); + cv[1] = gridVertex(i1,s*n2); + } + else + { + cv[0] = gridVertex((1-s)*n1, i0); + cv[1] = gridVertex((1-s)*n1, i1); + } + + const int cv0 = cv[0]; + int tv_int = -1; // Top-vertex interior to the master edge + int ki = -1; // Knot-span index of tv_int, w.r.t. the master edge + + if (lagTV) + { + tv_int = tvprev; + ki = kiprev; + } + + if (tvprev == -1) + { + kiprev = (i0 == 0 || i0 == ne) ? i1 : i0; + // Top-vertex interior to the master edge + tvprev = (i0 == 0 || i0 == ne) ? cv[1] : cv[0]; + } + else if (e_i < ne - 1) // Don't set to the endpoint + { + kiprev = (tvprev == cv[0]) ? i1 : i0; + // Next interior vertex along the master edge + tvprev = (tvprev == cv[0]) ? cv[1] : cv[0]; + } + + if (!lagTV) + { + tv_int = tvprev; + ki = kiprev; + } + + cv.Sort(); + if (cv[0] < 0) // may occur if gridVertex is not set everywhere. + { + continue; + } + else if (firstEdge == -1) + { + firstEdge = e_i; + if (e_i > 0) + { + tv_int = cv0; + ki = i0; + tvprev = cv.Sum() - cv0; // cv1 + kiprev = i1; + lagTV = true; + } + } + + const int tv = (e_idx == ne - de) ? -1 : tv_int; + const int tvki = (e_idx == ne - de) ? -1 : (reverse ? ne - ki : ki); + + const std::pair edge_i(cv[0], cv[1]); + const int childEdge = v2e.at(edge_i); + + if (tv == -1) { lagTV = true; } + + if (!parentVisited) + { + // edgePairs is ordered starting from the vertex of lower index. + edgePairs[edgePairOS[parentEdge] + e_idx].Set(tv, tvki, + childEdge, + parentEdge); + } + else + { + // Consistency check + const int os = edgePairOS[parentEdge]; + if (edgePairs[os + e_idx].child != childEdge || + edgePairs[os + e_idx].parent != parentEdge) + { + consistent = false; + } + } + } + + visitedParentEdges.insert(parentEdge); + } + } // dir + + // Set auxiliary and patch-slave faces outside the set gridVertex. Here, + // patch-slave refers to a slave face that is a face of a neighboring + // patch and may contain multiple mesh faces. In general, there can be at + // most 8 = 3^2 - 1 such faces. + + std::array gv1 = {0, r1min, r1max, n1}; + std::array gv2 = {0, r2min, r2max, n2}; + + if (hasSlaveFaces && !allset) + { + for (int i=0; i<3; ++i) + for (int j=0; j<3; ++j) + { + // Skip the middle, which is covered by gridVertex. + if (i == 1 && j == 1) { continue; } + + // Skip degenerate faces + if (gv1[i] == gv1[i+1] || gv2[j] == gv2[j+1]) { continue; } + + // Define auxiliary face (gv1[i], gv1[i+1]) x (gv2[j], gv2[j+1]) + SetFacePairOnGridRange(gv1[i], gv1[i+1], gv2[j], gv2[j+1]); + } + } + + // Set auxiliary edges outside the gridVertex, on the boundary of the + // parent face. Note that auxiliary edges cannot simply be found as edges + // of auxiliary faces, because the faces above are defined only on parent + // faces listed in V2K data. Other auxiliary faces will be defined in + // FindAdditionalFacesSA by using auxiliary edges found below. + + // Auxiliary edges in first and second directions + for (int d=0; d<2; ++d) + { + for (int i=0; i<3; ++i) + { + if (i == 1) // Skip the middle, which is covered by gridVertex. + { + continue; + } + + for (int s=0; s<2; ++s) // Loop over 2 sides in this direction + { + // Set gv1 (d == 0) or gv2 (d == 1) for this edge. + // Set range of set knot-span indices for this edge. + int rmin = -1; + int rmax = -1; + const int n_d = d == 0 ? n1 : n2; + for (int j=1; j= 0) + { + if (rmin == -1) + { + // Initialize range + rmin = j; + rmax = j; + } + else + { + rmin = std::min(rmin, j); + rmax = std::max(rmax, j); + } + } + } + + if (rmax == -1) + { + // No vertices set in gridVertex on the interior of this edge. + continue; + } + + if (d == 0) + { + gv1[1] = rmin; + gv1[2] = rmax; + } + else + { + gv2[1] = rmin; + gv2[2] = rmax; + } + + const int pid = d == 0 ? 2*s : (2*s) + 1; // Parent index + const bool reverse_p = parentEdgeRev[pid]; + // Sides with s=1 are reversed in defining parentEdgeRev. + const bool reverse = s == 0 ? reverse_p : !reverse_p; + + // Define an auxiliary edge (gv1[i], gv1[i+1]) + Array cv(2); + std::array ki; + + if (d == 0) + { + cv[0] = gridVertex(gv1[i],s*n2); + cv[1] = gridVertex(gv1[i+1],s*n2); + + ki[0] = gv1[i]; + ki[1] = gv1[i+1]; + } + else + { + cv[0] = gridVertex((1-s)*n1,gv2[i]); + cv[1] = gridVertex((1-s)*n1,gv2[i+1]); + + ki[0] = gv2[i]; + ki[1] = gv2[i+1]; + } + + if (cv[0] == cv[1]) + { + continue; + } + + // Top-vertex interior to the master edge. + const int tv = i == 0 ? cv[1] : cv[0]; + const int tvki_f = i == 0 ? ki[1] : ki[0]; // face index + const int tvki = reverse ? n_d - tvki_f : tvki_f; // edge index + + cv.Sort(); + MFEM_ASSERT(cv[0] >= 0, ""); + + const std::pair childPair(cv[0], cv[1]); + const bool childPairTopo = v2e.count(childPair) > 0; + if (!childPairTopo) + { + const int pv0 = d == 0 ? gridVertex(0,s*n2) : + gridVertex((1-s)*n1,0); + const int pv1 = d == 0 ? gridVertex(n1,s*n2) : + gridVertex((1-s)*n1,n2); + const std::pair parentPair(pv0 < pv1 ? pv0 : pv1, + pv0 < pv1 ? pv1 : pv0); + const int parentEdge = v2e.at(parentPair); + MFEM_ASSERT(parentEdges[pid] == parentEdge, ""); + + // Check whether childPair is in auxEdges. + if (auxv2e.count(childPair) == 0) + { + const int knotIndex0 = (d == 0) ? gv1[i] : gv2[i]; + const int knotIndex1 = (d == 0) ? gv1[i+1] : gv2[i+1]; + + // Create a new auxiliary edge + auxv2e[childPair] = auxEdges.size(); + auxEdges.emplace_back(AuxiliaryEdge{pv0 < pv1 ? + parentEdge : + -1 - parentEdge, + {childPair.first, childPair.second}, + {knotIndex0, knotIndex1}}); + } + + const bool start = (i == 0 && !reverse) || (i != 0 && reverse); + int end_idx = kvf_coarse.size() > 0 ? + (start ? 0 : kvf_coarse[kvi[d]].Size() - 1) : 0; + int de = kvf_coarse.size() > 0 ? kvf_coarse[kvi[d]][end_idx] : rf; + if (kvf.size() > 0 && kvf_coarse.size() == 0) + { + end_idx = start ? 0 : kvf[kvi[d]].Size() - 1; + de = kvf[kvi[d]][end_idx]; + } + + const int e_idx_i = i == 0 ? 0 : n_d - de; + const int e_idx = reverse ? n_d - de - e_idx_i : e_idx_i; + + const EdgePairInfo ep_e((e_idx == n_d - de) ? -1 : tv, + (e_idx == n_d - de) ? -1 : tvki, + -1 - auxv2e[childPair], parentEdge); + + const bool unset = !edgePairs[edgePairOS[parentEdge] + e_idx].isSet; + if (unset) + { + edgePairs[edgePairOS[parentEdge] + e_idx] = ep_e; + } + else + { + // Verify matching + MFEM_ASSERT(edgePairs[edgePairOS[parentEdge] + e_idx] == ep_e, ""); + } + } + else // childPairTopo == true, so this edge is a slave edge. + { + const int childEdge = v2e.at(childPair); + + const int pv0 = d == 0 ? gridVertex(0,s*n2) : gridVertex((1-s)*n1,0); + const int pv1 = d == 0 ? gridVertex(n1,s*n2) : gridVertex((1-s)*n1,n2); + const std::pair parentPair(pv0 < pv1 ? pv0 : pv1, + pv0 < pv1 ? pv1 : pv0); + const int parentEdge = v2e.at(parentPair); + MFEM_ASSERT(parentEdges[pid] == parentEdge, ""); + + const bool start = (i == 0 && !reverse) || (i != 0 && reverse); + int end_idx = kvf_coarse.size() > 0 ? + (start ? 0 : kvf_coarse[kvi[d]].Size() - 1) : 0; + int de = kvf_coarse.size() > 0 ? kvf_coarse[kvi[d]][end_idx] : rf; + if (kvf.size() > 0 && kvf_coarse.size() == 0) + { + end_idx = start ? 0 : kvf[kvi[d]].Size() - 1; + de = kvf[kvi[d]][end_idx]; + } + + const int e_idx_i = i == 0 ? 0 : n_d - de; + const int e_idx = reverse ? n_d - de - e_idx_i : e_idx_i; + + const int tv_e = (e_idx == n_d - de) ? -1 : tv; + const int tv_ki = (e_idx == n_d - de) ? -1 : tvki; + + const EdgePairInfo ep_e(tv_e, tv_ki, childEdge, parentEdge); + +#ifdef MFEM_DEBUG + const bool unset = + !edgePairs[edgePairOS[parentEdge] + e_idx].isSet; + const bool matching = + edgePairs[edgePairOS[parentEdge] + e_idx] == ep_e; + MFEM_ASSERT(unset || matching, ""); +#endif + + edgePairs[edgePairOS[parentEdge] + e_idx] = ep_e; + } + } + } + } + + if (hasSlaveFaces || hasAuxFace) { masterFaces.insert(parentFace); } + } // loop over parents + + MFEM_VERIFY(consistent, ""); +} + +void NCNURBSExtension::GetAuxFaceToPatchTable(Array2D &auxface2patch) +{ + auxface2patch.SetSize(auxFaces.size(), 2); + + if (auxFaces.size() == 0) { return; } + + auxface2patch = -1; + + const int dim = Dimension(); + + bool consistent = true; + + for (int p=0; p faces, orient; + if (dim == 2) { patchTopo->GetElementEdges(p, faces, orient); } + else { patchTopo->GetElementFaces(p, faces, orient); } + + for (auto face : faces) + { + const bool isMaster = dim == 2 ? masterEdgeToId.count(face) > 0 : + masterFaceToId.count(face) > 0; + if (isMaster) // If a master face + { + const int mid = dim == 2 ? masterEdgeToId.at(face) : + masterFaceToId.at(face); + const std::vector &slaves = dim == 2 ? masterEdgeInfo[mid].slaves : + masterFaceInfo[mid].slaves; + for (auto s : slaves) + { + if (s < 0) + { + // Auxiliary face. + const int aux = -1 - s; + if (auxface2patch(aux, 0) >= 0) + { + if (auxface2patch(aux, 1) != -1) { consistent = false; } + auxface2patch(aux, 1) = p; + } + else + { + auxface2patch(aux, 0) = p; + } + } + } + } + } + } + + MFEM_VERIFY(consistent, ""); +} + +void NCNURBSExtension::GetSlaveFaceToPatchTable(Array2D &sface2patch) +{ + const int dim = Dimension(); + const int numUnique = dim == 2 ? slaveEdgesUnique.Size() : + slaveFacesUnique.Size(); + sface2patch.SetSize(numUnique, 2); + + if (numUnique == 0) { return; } + + sface2patch = -1; + + bool consistent = true; + + for (int p=0; p faces, orient; + if (dim == 2) { patchTopo->GetElementEdges(p, faces, orient); } + else { patchTopo->GetElementFaces(p, faces, orient); } + + for (auto face : faces) + { + const bool isMaster = dim == 2 ? masterEdgeToId.count(face) > 0 : + masterFaceToId.count(face) > 0; + if (isMaster) // If a master face + { + const int mid = dim == 2 ? masterEdgeToId.at(face) : + masterFaceToId.at(face); + const std::vector &slaves = dim == 2 ? masterEdgeInfo[mid].slaves : + masterFaceInfo[mid].slaves; + for (auto id : slaves) + { + if (id >= 0) + { + const int s = dim == 2 ? slaveEdges[id] : slaveFaces[id].index; + const int u = dim == 2 ? slaveEdgesToUnique[s] : + slaveFacesToUnique[s]; + if (sface2patch(u, 0) >= 0) + { + if (sface2patch(u, 1) != -1) { consistent = false; } + sface2patch(u, 1) = p; + } + else + { + sface2patch(u, 0) = p; + } + } + } + } + } + } + + MFEM_VERIFY(consistent, ""); +} + +void RemapKnotIndex(bool rev, const Array &rf, int &k) +{ + const int ne = rf.Size(); + const int k0 = k; + k = 0; + for (int p=0; p &rf) +{ + for (auto auxEdge : auxEdges) + { + const int p = auxEdge.parent; + const int parent = p < 0 ? -1 - p : p; + const int kv = KnotInd(parent); + for (int i=0; i<2; ++i) + { + RemapKnotIndex(false, kvf[kv], auxEdge.ksi[i]); + } + } + + for (auto auxFace : auxFaces) + { + Array pv; + std::array quad; + patchTopo->GetFaceVertices(auxFace.parent, pv); + MFEM_ASSERT(pv.Size() == 4, ""); + for (int i=0; i<4; ++i) { quad[i] = pv[i]; } + // The face with vertices (pv0, pv1, pv2, pv3) is defined as a parent face. + const std::pair parentPair = QuadrupleToPair(quad); + const std::array kv = parentToKV.at(parentPair); + + RemapKnotIndex(false, kvf[kv[0]], auxFace.ksi0[0]); + RemapKnotIndex(false, kvf[kv[0]], auxFace.ksi1[0]); + + RemapKnotIndex(false, kvf[kv[1]], auxFace.ksi0[1]); + RemapKnotIndex(false, kvf[kv[1]], auxFace.ksi1[1]); + } +} + +const int NURBSExtension::unsetFactor; + +void NCNURBSExtension::LoadFactorsForKV(const std::string &filename) +{ + if (kvf_coarse.size() == 0) { kvf_coarse = kvf; } + if (kvf.size() == 0) { kvf.resize(NumOfKnotVectors); } + + for (int kv=0; kvGetNE()); + kvf[kv] = unsetFactor; + } + + if (filename.empty()) { return; } + + ifstream f(filename); + int nkv; + f >> nkv; + + for (int i=0; i> kv >> nf; + MFEM_ASSERT(nf == 1, ""); // TODO: support input of multiple factors. + + kvf[kv] = unsetFactor; + for (int j=0; j> rf; } + + for (int j=0; jGetNE(); + } + + return ki1 - ki0; +} + +// parentVerts are ordered with ascending knot-span index in parent edge, with +// knots from lower edge endpoint vertex to higher. +void NCNURBSExtension::SlaveEdgeToParent(int se, int parent, + const Array &os, + const std::vector &parentVerts, + Array &edges) +{ + Array sev(2); + if (se < 0) // Auxiliary edge + { + for (int i=0; i<2; ++i) { sev[i] = auxEdges[-1 - se].v[i]; } + } + else + { + patchTopo->GetEdgeVertices(se, sev); + } + + // Number of slave and auxiliary edges, not mesh edges + const int nedge = parentVerts.size() + 1; + MFEM_ASSERT((int) parentVerts.size() + 2 == os.Size(), ""); + + Array parentEndpoints; + patchTopo->GetEdgeVertices(parent, parentEndpoints); + + bool found = false; + for (int i=0; i &os) +{ + const int np = masterEdgeInfo[mid].slaves.size(); + MFEM_VERIFY(np > 0, ""); + os.SetSize(np + 1); + os[0] = 0; + + for (int i=0; i= 0) + { + nes = knotVectors[KnotInd(s)]->GetNE(); + } + else + { + nes = AuxiliaryEdgeNE(-1 - s); + } + + os[i+1] = os[i] + nes; + } +} + +Array CoarseToFineFactors(const Array &rf) +{ + Array frf(rf.Sum()); + int os = 0; + for (auto f : rf) + { + for (int i=0; i edges, oedges; + patchTopo->GetElementEdges(p, edges, oedges); + + const int dim = Dimension(); + const int nedge = dim == 3 ? 4 : 2; + + int dirSet = 0; + + bool partialChange = false; + bool consistent = true; + + auto SetFactorsDirection = [&](int j, int os_final, int af, Array &rf) + { + if (af == unsetFactor) { return; } + if (rf.Size() == 0) + { + rf.SetSize(os_final); + rf = unsetFactor; + } + if (rf[j] != unsetFactor && af != rf[j]) { consistent = false; } + rf[j] = af; + }; + + auto SetFactorsEdge = [&](int j, int rf, Array &pf) + { + if (rf == unsetFactor) { return; } + if (pf[j] != unsetFactor && pf[j] != rf) { consistent = false; } + pf[j] = rf; + }; + + auto LoopEdgesForDirection = [&](int d, Array &rf, bool first) + { + for (int i=0; iGetNE(); + const bool fullSize = kvf.size() > 0 && kvf[kv].Size() == nfe; + const Array rf_i = fullSize ? kvf[kv] : + CoarseToFineFactors(kvf[kv]); + if (rf_i.Size() > 0 && rf.Size() == 0) { rf = rf_i; } + } + else + { + if (kvf[kv] != rf) { partialChange = true; } + kvf[kv] = rf; + } + + if (isMaster) + { + // Check whether slave edges have factors set. + const int mid = masterEdgeToId.at(edge); + const int numPieces = masterEdgeInfo[mid].slaves.size(); + Array os; + GetMasterEdgePieceOffsets(mid, os); + + for (int piece=0; piece parentEdges; + Array *pf; // Refinement factors for this piece + + if (s >= 0) // Slave edge + { + const int kvs = KnotInd(s); + parentEdges.SetSize(kvf[kvs].Size()); + pf = &kvf[kvs]; + } + else // Aux edge + { + const int aux_edge = -1 - s; + if (auxef[aux_edge].Size() == 0) + { + auxef[aux_edge].SetSize(AuxiliaryEdgeNE(aux_edge)); + auxef[aux_edge] = unsetFactor; + } + parentEdges.SetSize(auxef[aux_edge].Size()); + pf = &auxef[aux_edge]; + } + + if (first && parentEdges.Size() == 0) { continue; } + SlaveEdgeToParent(s, edge, os, masterEdgeInfo[mid].vertices, parentEdges); + MFEM_ASSERT(parentEdges.Size() == os[piece + 1] - os[piece], ""); + + for (int j = os[piece]; j < os[piece + 1]; ++j) + { + const int jj = parentEdges[j - os[piece]]; + const int jr = rev ? rf.Size() - 1 - jj : jj; + if (first) + { + SetFactorsDirection(jr, os[numPieces], + (*pf)[j - os[piece]], rf); + } + else + { + SetFactorsEdge(j - os[piece], rf[jr], *pf); + } + } + } + } + } + }; + + for (int d=0; d rf; + LoopEdgesForDirection(d, rf, true); + if (rf.Size() == 0) { continue; } // This direction is unset + + // Set the same factor for all knotvectors in direction d. + LoopEdgesForDirection(d, rf, false); + if (rf.Min() > unsetFactor) { dirSet += pow(2, d); } + } + + MFEM_VERIFY(consistent, ""); + return partialChange ? -1 - dirSet : dirSet; +} + +void NCNURBSExtension::PropagateFactorsForKV(int rf_default) +{ + const int dim = Dimension(); + if (dim == 1 || num_structured_patches < 1) + { + for (size_t i=0; iGetFaceToElementTable(); + + Array2D auxface2patch, sface2patch; + GetAuxFaceToPatchTable(auxface2patch); + GetSlaveFaceToPatchTable(sface2patch); + + Array faces, orient; + + auto faceNeighbors = [&](int p, std::set &nghb) + { + if (dim == 2) { patchTopo->GetElementEdges(p, faces, orient); } + else { patchTopo->GetElementFaces(p, faces, orient); } + + for (auto face : faces) + { + Array row; + face2elem->GetRow(face, row); + + const bool isSlave = dim == 2 ? slaveEdgesToUnique.count( + face) > 0 : slaveFacesToUnique.count(face) > 0; + if (isSlave) + { + const int u = dim == 2 ? slaveEdgesToUnique[face] : + slaveFacesToUnique[face]; + for (int i=0; i<2; ++i) + { + const int elem = sface2patch(u, i); + if (elem >= 0 && elem != p) { nghb.insert(elem); } + } + } + + for (auto elem : row) { nghb.insert(elem); } + } + }; + + auto masterFaceNeighbors = [&](int p, std::set &nghb) + { + if (dim == 2) { patchTopo->GetElementEdges(p, faces, orient); } + else { patchTopo->GetElementFaces(p, faces, orient); } + + for (auto face : faces) + { + const bool isMaster = dim == 2 ? masterEdgeToId.count(face) > 0 : + masterFaceToId.count(face) > 0; + if (isMaster) // If a master face + { + const int mid = dim == 2 ? masterEdgeToId.at(face) : + masterFaceToId.at(face); + const std::vector &slaves = + dim == 2 ? masterEdgeInfo[mid].slaves : masterFaceInfo[mid].slaves; + for (auto s : slaves) + { + if (s < 0) + { + // Auxiliary face. + const int aux = -1 - s; + for (int i=0; i<2; ++i) + { + const int patch = auxface2patch(aux, i); + if (patch >= 0) { nghb.insert(patch); } + } + } + else + { + // Slave face in patchTopo. + Array row; + face2elem->GetRow(s, row); + for (auto elem : row) { nghb.insert(elem); } + } + } + } + } + }; + + const int npatchall = patches.Size(); + Array patchState(npatchall); + patchState = 0; + + auxef.resize(auxEdges.size()); + + std::set nextPatches, unchanged; + const int dirAllSet = dim == 3 ? 7 : 3; + int lastChanged = 0; + int iter = 0; + bool done = false; + while (iter < 100 && !done) + { + // Start each iteration at the patch last changed + nextPatches.clear(); + nextPatches.insert(lastChanged); + + std::set visited; // Visit each patch only once per iteration + iter++; + + while (nextPatches.size() > 0) + { + const int p = *nextPatches.begin(); + nextPatches.erase(p); + + visited.insert(p); + + const int dirSetSigned = SetPatchFactors(p); + const bool partialChange = dirSetSigned < 0; + const int dirSet = partialChange ? -1 - dirSetSigned : dirSetSigned; + const bool changed = (patchState[p] != dirSet) || partialChange; + patchState[p] = dirSet; + + // Find neighbors of patch p + std::set neighbors; + + // First, find neighbors sharing a conforming face, via face2elem. + faceNeighbors(p, neighbors); + + // Second, find neighbors sharing a slave/auxiliary face in patchTopo. + masterFaceNeighbors(p, neighbors); + + // Add neighbors not done to nextPatches. Note that a patch can be + // added to nextPatches on multiple iterations, to propagate factors in + // different directions, on multiple sweeps. + + for (auto n : neighbors) + { + if (n < npatchall && n != p && patchState[n] != dirAllSet && + visited.count(n) == 0) + { + nextPatches.insert(n); + } + } + + if (changed) + { + unchanged.erase(p); + lastChanged = p; + } + else + { + unchanged.insert(p); + } + + if (unchanged.size() == (size_t) npatchall) + { + // Make another pass through all patches to check for changes + for (int i=0; iGetNE()); + kvf[i] = rf_default; + } + else + { + for (int j=0; jspacing) + { + PiecewiseSpacingFunction *pws = dynamic_cast + (knotVectors[i]->spacing.get()); + if (pws) + { + Array pwn = pws->RelativePieceSizes(); + const bool rev = pws->GetReverse(); + const int np = pwn.Size(); + const int f = kvf[i].Size() / pwn.Sum(); + MFEM_ASSERT(kvf[i].Size() == f * pwn.Sum(), ""); + + Array os(np + 1); + os[0] = 0; + for (int j=1; j pwf(np); + for (int j=0; jScalePartition(pwf, true); + } + } + } +} + +void UpdateFactors(Array &f) +{ + Array rf(f.Sum()); + + int os = 0; + for (int i=0; i 0) + { + MFEM_VERIFY(ref_factors.Size() == Dimension(), ""); + for (int i=0; i> num_structured_patches; + + const int maxOrder = mOrders.Max(); + + // For degree maxOrder, there are 2*(maxOrder + 1) knots for a single + // element, and the number of control points in each dimension is + // 2*(maxOrder + 1) - maxOrder - 1 + const int ncp1D = maxOrder + 1; + const int ncp = pow(ncp1D, Dimension()); + + patchCP.SetSize(num_structured_patches, ncp, Dimension()); + for (int p=0; p> patchCP(p, i, j); } +} + +void NCNURBSExtension::PrintCoarsePatches(std::ostream &os) +{ + const int maxOrder = mOrders.Max(); + const int patchCP_size1 = patchCP.GetSize1(); + MFEM_VERIFY(patchCP_size1 == num_structured_patches || patchCP_size1 == 0, + ""); + + if (patchCP_size1 == 0) { return; } + + // For degree maxOrder, there are 2*(maxOrder + 1) knots for a single element, + // and the number of control points in each dimension is + // 2*(maxOrder + 1) - maxOrder - 1 + const int ncp1D = maxOrder + 1; + const int ncp = pow(ncp1D, Dimension()); + + os << "\npatch_cp\n" << num_structured_patches << "\n"; + for (int p=0; p &f, Array &c) +{ + MFEM_ASSERT(f.Size() == c.Sum(), ""); + bool consistent = true; + int os = 0; + for (int j=0; j &verts) +{ + Array fverts; + mesh->GetFaceVertices(face, fverts); + MFEM_ASSERT(fverts.Size() == 4, ""); + + // Verify that verts and fvert have the same entries as sets, by deep-copying + // and sorting. + { + Array s1(4); + Array s2(fverts); + + for (int i=0; i<4; ++i) { s1[i] = verts[i]; } + + s1.Sort(); s2.Sort(); + MFEM_ASSERT(s1 == s2, ""); + } + + // Find the shift of the first vertex. + int s = -1; + for (int i=0; i<4; ++i) + { + if (verts[i] == fverts[0]) { s = i; } + } + + // Check whether ordering is reversed. + const bool rev = verts[(s + 1) % 4] != fverts[1]; + if (rev) { s = -1 - s; } // Reversed order is encoded by the sign. + return s; +} + +// The 2D array `a` is of size n1*n2, with index j + n2*i corresponding to (i,j) +// with the fast index j, for 0 <= i < n1 and 0 <= j < n2. We assume that j is +// the fast index in (i,j). The orientation is encoded by ori, defining a shift +// and relative direction, such that a quad face F1, on which the ordering of +// `a` is based, has vertex with index `shift` matching vertex 0 of the new quad +// face F2, on which the new ordering of `a` should be based. For more details, +// see GetFaceOrientation. +bool Reorder2D(int ori, std::array &s0) +{ + const int shift = ori < 0 ? -1 - ori : ori; + + // Shift is an F1 index in the counter-clockwise ordering of 4 quad vertices. + // Now find the (i,j) indices of this index, with i,j in {0,1}. + const int s0i = (shift == 0 || shift == 3) ? 0 : 1; + const int s0j = (shift < 2) ? 0 : 1; + + s0[0] = s0i; + s0[1] = s0j; + + // Determine whether the dimensions of F1 and F2 are reversed. Do this by + // finding the (i,j) indices of s1, which is the next vertex on F1. + const int shift1 = ori < 0 ? shift - 1: shift + 1; + const int s1 = (shift1 + 4) % 4; + const int s1i = (s1 == 0 || s1 == 3) ? 0 : 1; + const bool dimReverse = s0i == s1i; + + return dimReverse; +} + +void GetInverseShiftedDimensions2D(int signedShift, int sm, int sn, int &m, + int &n) +{ + const bool rev = (signedShift < 0); + const int shift = rev ? -1 - signedShift : signedShift; + MFEM_ASSERT(0 <= shift && shift < 4, ""); + + // We consider 8 cases for the possible values of rev and shift. + if (rev) + { + if (shift == 0) + { + // New: 3 2 Old: 1 2 + // 0 1 0 3 + n = sm; + m = sn; + } + else if (shift == 1) + { + // New: 3 2 Old: 2 3 + // 0 1 1 0 + m = sm; + n = sn; + } + else if (shift == 2) + { + // New: 3 2 Old: 3 0 + // 0 1 2 1 + n = sm; + m = sn; + } + else // shift == 3 + { + // New: 3 2 Old: 0 1 + // 0 1 3 2 + m = sm; + n = sn; + } + } + else + { + if (shift == 0) + { + // New: 3 2 Old: 3 2 + // 0 1 0 1 + m = sm; + n = sn; + } + else if (shift == 1) + { + // New: 3 2 Old: 0 3 + // 0 1 1 2 + n = sm; + m = sn; + } + else if (shift == 2) + { + // New: 3 2 Old: 1 0 + // 0 1 2 3 + m = sm; + n = sn; + } + else // shift == 3 + { + // New: 3 2 Old: 2 1 + // 0 1 3 0 + n = sm; + m = sn; + } + } +} + +void GetShiftedGridPoints2D(int m, int n, int i, int j, int signedShift, + int& sm, int& sn, int& si, int& sj) +{ + const bool rev = (signedShift < 0); + const int shift = rev ? -1 - signedShift : signedShift; + MFEM_ASSERT(0 <= shift && shift < 4, ""); + + // (0,0) <= (i,j) < (m,n) are old indices, and old vertex [shift] maps + // to new vertex 0 in counter-clockwise quad ordering. + + // We consider 8 cases for the possible values of rev and shift. + if (rev) + { + if (shift == 0) + { + // New: 3 2 Old: 1 2 + // 0 1 0 3 + sm = n; + sn = m; + + si = j; + sj = i; + } + else if (shift == 1) + { + // New: 3 2 Old: 2 3 + // 0 1 1 0 + sm = m; + sn = n; + + si = m - 1 - i; + sj = j; + } + else if (shift == 2) + { + // New: 3 2 Old: 3 0 + // 0 1 2 1 + sm = n; + sn = m; + + si = n - 1 - j; + sj = m - 1 - i; + } + else // shift == 3 + { + // New: 3 2 Old: 0 1 + // 0 1 3 2 + sm = m; + sn = n; + + si = i; + sj = n - 1 - j; + } + } + else + { + if (shift == 0) + { + // New: 3 2 Old: 3 2 + // 0 1 0 1 + sm = m; + sn = n; + + si = i; + sj = j; + } + else if (shift == 1) + { + // New: 3 2 Old: 0 3 + // 0 1 1 2 + sm = n; + sn = m; + + si = j; + sj = m - 1 - i; + } + else if (shift == 2) + { + // New: 3 2 Old: 1 0 + // 0 1 2 3 + sm = m; + sn = n; + + si = m - 1 - i; + sj = n - 1 - j; + } + else // shift == 3 + { + // New: 3 2 Old: 2 1 + // 0 1 3 0 + sm = n; + sn = m; + + si = n - 1 - j; + sj = i; + } + } +} + +// Given a quadruple in q, return the pair (q_i, q_j), where q_i is the minimum +// entry of q, and q_j is the entry two indices away from q_i. When q contains +// indices of the vertices of a quadrilateral, the returned pair represents the +// unique diagonal touching the vertex of minimum index, which is a more concise +// way of representing the quadrilateral, facilitating the search of faces. +std::pair QuadrupleToPair(const std::array &q) +{ + const auto qmin = std::min_element(q.begin(), q.end()); + const int idmin = std::distance(q.begin(), qmin); + return std::pair(q[idmin], q[(idmin + 2) % 4]); +} + +void VertexToKnotSpan::SetSize(int dimension, int numVertices) +{ + dim = dimension; + MFEM_ASSERT((dim == 2 || dim == 3) && numVertices > 0, "Invalid size"); + data.SetSize(numVertices, dim == 3 ? 7 : 4); +} + +void VertexToKnotSpan::SetVertex2D(int index, int v, int ks, + const std::array &pv) +{ + data(index,0) = v; + data(index,1) = ks; + data(index,2) = pv[0]; + data(index,3) = pv[1]; +} + +void VertexToKnotSpan::SetVertex3D(int index, int v, + const std::array &ks, + const std::array &pv) +{ + data(index,0) = v; + data(index,1) = ks[0]; + data(index,2) = ks[1]; + data(index,3) = pv[0]; + data(index,4) = pv[1]; + data(index,5) = pv[2]; + data(index,6) = pv[3]; +} + +void VertexToKnotSpan::SetKnotSpan2D(int index, int ks) +{ + data(index,1) = ks; +} + +void VertexToKnotSpan::SetKnotSpans3D(int index, const std::array &ks) +{ + data(index,1) = ks[0]; + data(index,2) = ks[1]; +} + +void VertexToKnotSpan::GetVertex2D(int index, int &v, int &ks, + std::array &pv) const +{ + v = data(index,0); + ks = data(index,1); + pv[0] = data(index,2); + pv[1] = data(index,3); +} + +void VertexToKnotSpan::GetVertex3D(int index, int &v, std::array &ks, + std::array &pv) const +{ + v = data(index,0); + ks[0] = data(index,1); + ks[1] = data(index,2); + pv[0] = data(index,3); + pv[1] = data(index,4); + pv[2] = data(index,5); + pv[3] = data(index,6); +} + +void VertexToKnotSpan::Print(std::ostream &os) const +{ + const int nv = data.NumRows(); + const int m = data.NumCols(); + os << nv << "\n"; + for (int i = 0; i < nv; i++) + { + os << data(i,0); + for (int j = 1; j < m; j++) + { + os << " " << data(i,j); + } + os << "\n"; + } +} + +std::pair VertexToKnotSpan::GetVertexParentPair(int index) const +{ + if (dim == 3) + { + std::array pv; + for (int i=0; i<4; ++i) { pv[i] = data(index, 3 + i); } + // The face with vertices (pv[0], pv[1], pv[2], pv[3]) is defined as a + // parent face. + return QuadrupleToPair(pv); + } + + int c0 = data(index, 2); + int c1 = data(index, 3); + if (c0 > c1) { std::swap(c0, c1); } + return std::pair(c0, c1); +} + +void NCNURBSExtension::UniformRefinement(const Array &rf) +{ + MFEM_VERIFY(!nonconformingPT, + "NURBS NC-patch meshes cannot use this method of refinement"); + + if (ref_factors.Size()) + { + MFEM_VERIFY(ref_factors.Size() == rf.Size(), ""); + for (int i=0; i *rf) +{ + const int maxOrder = mOrders.Max(); + const int dim = Dimension(); + + for (int p = 0; p < patches.Size(); p++) + { + if (nonconformingPT) + { + std::vector> prf(dim); + Array pkv(dim); + Array edges, orient; + patchTopo->GetElementEdges(p, edges, orient); + + if (dim == 3) + { + constexpr char e3[3] = {0, 3, 8}; + for (int i=0; i<3; ++i) + { + prf[i] = kvf[KnotInd(edges[e3[i]])]; + pkv[i] = knotVectors[KnotInd(edges[e3[i]])]; + } + } + else + { + MFEM_VERIFY(dim == 2, ""); + for (int i=0; i<2; ++i) + { + prf[i] = kvf[KnotInd(edges[i])]; + pkv[i] = knotVectors[KnotInd(edges[i])]; + } + } + + if (p >= num_structured_patches) + { + for (int i=0; iUpdateSpacingPartitions(pkv); + patches[p]->UniformRefinement(prf, coarsened, maxOrder); + } + else + { + patches[p]->UniformRefinement(*rf); + } + } + + if (nonconformingPT) + { + patchTopo->ncmesh->RefineVertexToKnotSpan(kvf, knotVectors, parentToKV); + UpdateAuxiliaryKnotSpans(ref_factors); + UpdateCoarseKVF(); + } +} + +void NCNURBSExtension::SetDofToPatch() +{ + dof2patch.SetSize(NumOfDofs); + dof2patch = -1; + + const int dim = Dimension(); + if (dim == 1) { return; } + + Array edges, faces, orient; + const int np = patchTopo->GetNE(); + + for (int p = 0; p < np; p++) + { + patchTopo->GetElementEdges(p, edges, orient); + for (auto e : edges) + { + if (masterEdges.count(e) > 0) + { + Array mdof; + GetMasterEdgeDofs(true, e, mdof); + for (auto dof : mdof) { dof2patch[dof] = p; } + } + } + + if (dim == 3) + { + patchTopo->GetElementFaces(p, faces, orient); + + for (auto f : faces) + { + if (masterFaces.count(f) > 0) + { + Array2D mdof; + GetMasterFaceDofs(true, f, mdof); + for (int j=0; jGetNE(); + + // Total number of CP on edge, excluding vertex CP. + const int totalEdgeCP = kv->GetNCP() - 2 - ne + 1; + const int perEdgeCP = totalEdgeCP / ne; + + MFEM_VERIFY(perEdgeCP * ne == totalEdgeCP, ""); + + return perEdgeCP; +} + +void NCNURBSExtension::GenerateOffsets() +{ + const int nv = patchTopo->GetNV(); + const int ne = patchTopo->GetNEdges(); + const int nf = patchTopo->GetNFaces(); + const int np = patchTopo->GetNE(); + int meshCounter, spaceCounter, dim = Dimension(); + + std::set reversedParents; + if (patchTopo->ncmesh) + { + // Note that master or slave entities exist only for a mesh with + // vertex_parents, not for the vertex_to_knotspan case. Currently, a mesh + // is not allowed to have both cases, see the MFEM_VERIFY below. + + const NCMesh::NCList& nce = patchTopo->ncmesh->GetNCList(1); + const NCMesh::NCList& ncf = patchTopo->ncmesh->GetNCList(2); + + masterEdges.clear(); + masterFaces.clear(); + slaveEdges.clear(); + slaveFaces.clear(); + masterEdgeToId.clear(); + masterFaceToId.clear(); + + MFEM_VERIFY(nce.masters.Size() > 0 || + patchTopo->ncmesh->GetVertexToKnotSpan().Size() > 0, ""); + MFEM_VERIFY(!(nce.masters.Size() > 0 && + patchTopo->ncmesh->GetVertexToKnotSpan().Size() > 0), ""); + + std::vector edgePairs; + std::vector facePairs; + std::vector parentFaces, parentVerts; + std::vector> parentSize; + + const bool is3D = dim == 3; + + std::map, int> v2f; + + if (patchTopo->ncmesh->GetVertexToKnotSpan().Size() > 0) + { + // Intersections of master edges may not be edges in patchTopo->ncmesh, + // so we represent them in auxEdges, to account for their vertices and + // DOFs. + { + int vert_index[2]; + const NCMesh::NCList& EL = patchTopo->ncmesh->GetEdgeList(); + for (auto edgeID : EL.conforming) + { + patchTopo->ncmesh->GetEdgeVertices(edgeID, vert_index); + v2e[std::pair (vert_index[0], vert_index[1])] = edgeID.index; + } + } + + if (is3D) + { + Array vert; + for (int i=0; iGetNumFaces(); ++i) + { + patchTopo->GetFaceVertices(i, vert); + const int vmin = vert.Min(); + const int idmin = vert.Find(vmin); + v2f[std::pair (vert[idmin], vert[(idmin + 2) % 4])] = i; + } + } + + const VertexToKnotSpan &v2k = patchTopo->ncmesh->GetVertexToKnotSpan(); + + if (is3D) + ProcessVertexToKnot3D(v2k, v2f, parentSize, edgePairs, + facePairs, parentFaces, parentVerts); + else + { + ProcessVertexToKnot2D(v2k, reversedParents, edgePairs); + } + } // if using vertex_to_knotspan + + const int numMasters = is3D ? ncf.masters.Size() : nce.masters.Size(); + + if (is3D) + { + for (auto masterFace : ncf.masters) + { + masterFaces.insert(masterFace.index); + } + } + + for (auto masterEdge : nce.masters) + { + masterEdges.insert(masterEdge.index); + } + + masterEdgeIndex.SetSize(masterEdges.size()); + int cnt = 0; + for (auto medge : masterEdges) + { + masterEdgeIndex[cnt] = medge; + masterEdgeToId[medge] = cnt; + cnt++; + } + MFEM_VERIFY(cnt == masterEdgeIndex.Size(), ""); + + Array masterFaceIndex(parentFaces.size()); + + // Note that masterFaces is a subset of parentFaces. + MFEM_VERIFY(masterFaces.size() <= parentFaces.size(), ""); + + cnt = 0; + for (auto mface : parentFaces) + { + masterFaceIndex[cnt] = mface; + masterFaceToId[mface] = cnt; + cnt++; + } + + MFEM_VERIFY(cnt == masterFaceIndex.Size(), ""); + + masterEdgeInfo.clear(); + masterEdgeInfo.resize(masterEdgeIndex.Size()); + + masterFaceInfo.clear(); + masterFaceInfo.resize(masterFaceIndex.Size()); + + if (patchTopo->ncmesh->GetVertexToKnotSpan().Size() > 0) + { + // Note that this is used in 2D and 3D. + const int npairs = edgePairs.size(); + + for (int i=0; i= 0) + { + masterEdgeInfo[mid].vertices.push_back(v); + masterEdgeInfo[mid].ks.push_back(ksi); + } + } + + ProcessFacePairs(0, 0, parentSize, parentVerts, facePairs); + } + + for (int i=0; incmesh->GetEdgeVertices(slaveEdge, vert_index); + slaveEdges.push_back(slaveEdge.index); + + const int mid = masterEdgeToId[slaveEdge.master]; + masterEdgeInfo[mid].slaves.push_back(i); + } + + if (!is3D) + { + for (int m=0; m 0, ""); + int mvert[2]; + int svert[2]; + patchTopo->ncmesh->GetEdgeVertices(nce.masters[m], mvert); + + std::vector orderedSlaves(numSlaves); + std::set used; + + int vi = mvert[0]; + for (int s=0; s 0) { continue; } + patchTopo->ncmesh->GetEdgeVertices(nce.slaves[sid], svert); + if (svert[0] == vi || svert[1] == vi) + { + orderedSlaves[s] = sid; + used.insert(sid); + break; + } + } + + MFEM_ASSERT(orderedSlaves[s] >= 0, ""); + + // Update vi to the next vertex + vi = (svert[0] == vi) ? svert[1] : svert[0]; + + if (s < numSlaves - 1) + { + masterEdgeInfo[m].vertices.push_back(vi); + masterEdgeInfo[m].ks.push_back(-1); // Used only in 3D. + } + } + + masterEdgeInfo[m].slaves = orderedSlaves; + } // m + } + + if (is3D) + { + // Remove edges from masterEdges if they do not have any slave edges. + std::vector falseMasterEdges; + for (auto me : masterEdges) + { + const int mid = masterEdgeToId.at(me); + if (masterEdgeInfo[mid].slaves.size() <= 1) + { + falseMasterEdges.push_back(me); + } + } + + for (auto me : falseMasterEdges) { masterEdges.erase(me); } + + // Find slave and auxiliary faces not yet defined. + const int nfp0 = facePairs.size(); + std::set addParentFaces; + FindAdditionalFacesSA(v2f, addParentFaces, facePairs); + + cnt = parentFaces.size(); + const int npf0 = cnt; + + for (auto pf : addParentFaces) + { + if (masterFaces.count(pf) == 0) + { + masterFaces.insert(pf); + masterFaceIndex.Append(pf); + + masterFaceToId[pf] = cnt; + cnt++; + + { + Array edges, ori, verts; + patchTopo->GetFaceEdges(pf, edges, ori); + patchTopo->GetFaceVertices(pf, verts); + MFEM_ASSERT(edges.Size() == 4 && verts.Size() == 4, ""); + + parentSize.emplace_back(std::array + { + KnotVec(edges[0])->GetNE(), + KnotVec(edges[1])->GetNE() + }); + + masterFaceInfo.push_back( + MasterFaceInfo(KnotVec(edges[0])->GetNE(), + KnotVec(edges[1])->GetNE())); + + for (int i=0; i<4; ++i) { parentVerts.push_back(verts[i]); } + } + } + } + + MFEM_VERIFY(cnt == masterFaceIndex.Size(), ""); + + ProcessFacePairs(nfp0, npf0, parentSize, parentVerts, facePairs); + } + } + + for (auto rp : reversedParents) + { + masterEdgeInfo[masterEdgeToId[rp]].Reverse(); + } + + Array edges, orient; + + v_meshOffsets.SetSize(nv); + e_meshOffsets.SetSize(ne); + f_meshOffsets.SetSize(nf); + p_meshOffsets.SetSize(np); + + v_spaceOffsets.SetSize(nv); + e_spaceOffsets.SetSize(ne); + f_spaceOffsets.SetSize(nf); + p_spaceOffsets.SetSize(np); + + // Get vertex offsets + for (meshCounter = 0; meshCounter < nv; meshCounter++) + { + v_meshOffsets[meshCounter] = meshCounter; + v_spaceOffsets[meshCounter] = meshCounter; + } + spaceCounter = meshCounter; + + // Get edge offsets + for (int e = 0; e < ne; e++) + { + e_meshOffsets[e] = meshCounter; + e_spaceOffsets[e] = spaceCounter; + + if (masterEdges.count(e) == 0) // If not a master edge + { + meshCounter += KnotVec(e)->GetNE() - 1; + spaceCounter += KnotVec(e)->GetNCP() - 2; + } + } + + const int nauxe = auxEdges.size(); + aux_e_meshOffsets.SetSize(nauxe + 1); + aux_e_spaceOffsets.SetSize(nauxe + 1); + for (int e = 0; e < nauxe; e++) + { + aux_e_meshOffsets[e] = meshCounter; + aux_e_spaceOffsets[e] = spaceCounter; + + // Find the number of elements and CP in this auxiliary edge, which is + // defined only on part of the master edge knotvector. + const int signedParentEdge = auxEdges[e].parent; + const int ki0 = auxEdges[e].ksi[0]; + const int ki1raw = auxEdges[e].ksi[1]; + const bool rev = signedParentEdge < 0; + const int parentEdge = rev ? -1 - signedParentEdge : signedParentEdge; + const int masterNE = KnotVec(parentEdge)->GetNE(); + const int ki1 = ki1raw == -1 ? masterNE : ki1raw; + const int perEdgeCP = GetNCPperEdge(KnotVec(e)); + const int auxne = ki1 - ki0; + MFEM_ASSERT(auxne > 0, ""); + meshCounter += auxne - 1; + spaceCounter += (auxne * perEdgeCP) + auxne - 1; + } + + aux_e_meshOffsets[nauxe] = meshCounter; + aux_e_spaceOffsets[nauxe] = spaceCounter; + + // Get face offsets + for (int f = 0; f < nf; f++) + { + f_meshOffsets[f] = meshCounter; + f_spaceOffsets[f] = spaceCounter; + + if (masterFaces.count(f) == 0) // If not a master face + { + patchTopo->GetFaceEdges(f, edges, orient); + + meshCounter += + (KnotVec(edges[0])->GetNE() - 1) * + (KnotVec(edges[1])->GetNE() - 1); + spaceCounter += + (KnotVec(edges[0])->GetNCP() - 2) * + (KnotVec(edges[1])->GetNCP() - 2); + } + } + + const int nauxf = auxFaces.size(); + aux_f_meshOffsets.SetSize(nauxf + 1); + aux_f_spaceOffsets.SetSize(nauxf + 1); + for (int f = 0; f < nauxf; f++) + { + aux_f_meshOffsets[f] = meshCounter; + aux_f_spaceOffsets[f] = spaceCounter; + + const int parentFace = auxFaces[f].parent; + patchTopo->GetFaceEdges(parentFace, edges, orient); + + // Number of control points per edge, in first and second directions. + const int perEdgeCP0 = GetNCPperEdge(KnotVec(edges[0])); + const int perEdgeCP1 = GetNCPperEdge(KnotVec(edges[1])); + + const int auxne0 = auxFaces[f].ksi1[0] - auxFaces[f].ksi0[0]; + const int auxne1 = auxFaces[f].ksi1[1] - auxFaces[f].ksi0[1]; + meshCounter += (auxne0 - 1) * (auxne1 - 1); + spaceCounter += ((auxne0 * perEdgeCP0) + auxne0 - 1) * + ((auxne1 * perEdgeCP1) + auxne1 - 1); + } + + aux_f_meshOffsets[nauxf] = meshCounter; + aux_f_spaceOffsets[nauxf] = spaceCounter; + + // Get patch offsets + GetPatchOffsets(meshCounter, spaceCounter); + + NumOfVertices = meshCounter; + NumOfDofs = spaceCounter; + + SetDofToPatch(); +} + +} // namespace mfem diff --git a/mesh/ncnurbs.hpp b/mesh/ncnurbs.hpp new file mode 100644 index 0000000000..106241eb71 --- /dev/null +++ b/mesh/ncnurbs.hpp @@ -0,0 +1,329 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_NCNURBS +#define MFEM_NCNURBS + +#include "nurbs.hpp" + +namespace mfem +{ + +/** @brief NCNURBSExtension extends NURBSExtension to support NC-patch NURBS + meshes. */ +class NCNURBSExtension : public NURBSExtension +{ +public: + /// Copy constructor: deep copy + NCNURBSExtension(const NCNURBSExtension &orig); + + NCNURBSExtension(std::istream &input, bool spacing=false); + + void UniformRefinement(const Array &rf) override; + +protected: + /** @brief Set the mesh and space offsets, and also count the global + @a NumOfVertices and the global @a NumOfDofs. */ + void GenerateOffsets() override; + + /// Return true if @a edge is a master NC-patch edge. + bool IsMasterEdge(int edge) const override + { return masterEdges.count(edge) > 0; } + + /// Return true if @a face is a master NC-patch face. + bool IsMasterFace(int face) const override + { return masterFaces.count(face) > 0; } + + /// Given a pair of vertices, return the corresponding edge. + int VertexPairToEdge(const std::pair &vertices) const override + { return v2e.at(vertices); } + + /** @brief Get the DOFs (dof = true) or vertices (dof = false) for + master edge @a me. */ + void GetMasterEdgeDofs(bool dof, int me, Array &dofs) const override; + + /** @brief Get the DOFs (dof = true) or vertices (dof = false) for + master face @a mf. */ + void GetMasterFaceDofs(bool dof, int mf, Array2D &dofs) const override; + + /// Load refinement factors for a list of knotvectors from file. + void LoadFactorsForKV(const std::string &filename); + + /// Set consistent refinement factors on patch @a p. + int SetPatchFactors(int p); + + /// Ensure consistent refinement factors on all knotvectors. + void PropagateFactorsForKV(int rf_default); + + /// Refine with refinement factors loaded for some knotvectors specified in + /// the given file, with default refinement factor @a rf elsewhere. The flag + /// @a coarsened indicates whether each patch is a single element. + void RefineWithKVFactors(int rf, const std::string &kvf_filename, + bool coarsened) override; + +private: + /// Global mesh offsets, meshOffsets == meshVertexOffsets + Array aux_e_meshOffsets, aux_f_meshOffsets; + + /// Global space offsets, spaceOffsets == dofOffsets + Array aux_e_spaceOffsets, aux_f_spaceOffsets; + + /// Represents a nonconforming edge not in patchTopo->ncmesh. + struct AuxiliaryEdge + { + int parent; /// Signed parent edge index (sign encodes orientation) + int v[2]; /// Vertex indices + int ksi[2]; /// Knot-span indices of vertices in parent edge + }; + + /// Represents a nonconforming face not in patchTopo->ncmesh. + struct AuxiliaryFace + { + int parent; /// Parent face index + int ori; /// Orientation with respect to parent face + int v[4]; /// Vertex indices + int ksi0[2]; /// Lower knot-span indices in parent face + int ksi1[2]; /// Upper knot-span indices in parent face + }; + + /** @brief Represents a pair of child and parent edges for a nonconforming + patch topology. */ + struct EdgePairInfo + { + int v; /// Vertex index + int ksi; /// Knot-span index of vertex + int child, parent; /// Child and parent edge indices + bool isSet; /// Whether this instance is set + + EdgePairInfo() : isSet(false) { } + + EdgePairInfo(int vertex, int knotIndex, int childEdge, int parentEdge) + : v(vertex), ksi(knotIndex), child(childEdge), parent(parentEdge), + isSet(true) { } + + /// Set the data members. + void Set(int vertex, int knotIndex, int childEdge, int parentEdge) + { + v = vertex; + ksi = knotIndex; + child = childEdge; + parent = parentEdge; + isSet = true; + } + + bool operator==(const EdgePairInfo& other) const + { + return v == other.v && ksi == other.ksi && child == other.child + && parent == other.parent; + } + }; + + /// Master edge data for a nonconforming patch topology. + struct MasterEdgeInfo + { + std::vector slaves; /// Slave edge indices on the master edge + std::vector vertices; /// Vertex indices on the master edge + std::vector ks; /// Knot-span indices of vertices on the master edge + + void Reverse() + { + std::reverse(slaves.begin(), slaves.end()); + std::reverse(vertices.begin(), vertices.end()); + std::reverse(ks.begin(), ks.end()); + } + }; + + /// Master face data for a nonconforming patch topology. + struct MasterFaceInfo + { + std::vector slaves; /// Slave face indices on the master face + std::vector slaveCorners; /// Corner vertices of slave faces + std::array ne; /// Number of elements in each direction + std::array s0; /// Cartesian shift, see Reorder2D + bool rev; /// Whether dimensions are interchanged + + MasterFaceInfo() : rev(false) + { + for (int i=0; i<2; ++i) + { + ne[i] = 0; + s0[i] = -1; + } + } + + MasterFaceInfo(int ne1, int ne2) : rev(false) + { + ne[0] = ne1; + ne[1] = ne2; + for (int i=0; i<2; ++i) { s0[i] = -1; } + } + }; + + /// Slave face data for a nonconforming patch topology. + struct SlaveFaceInfo + { + int index; /// Face index + int ori; /// Orientation + int ksi[2]; /// Knot-span indices in parent face of v0 + int ne[2]; /// Number of elements in each direction on child face + }; + + /** @brief Represents a pair of child and parent faces for a nonconforming + patch topology. */ + struct FacePairInfo + { + int v0; /// Lower left corner vertex + int parent; /// Parent face index + SlaveFaceInfo info; /// Data for the child face + }; + + /// Auxiliary edges and faces for a nonconforming patch topology. + std::vector auxEdges; + std::vector auxFaces; + + /** @brief Maps from vertex pairs to indices in auxEdges, auxFaces. Vertex + pairs are sorted indices, with faces having 4 vertices represented by the + minimum index and the index of the diagonally opposite vertex. */ + std::map, int> auxv2e, auxv2f; + + /// Map from sorted vertex pairs to edge indices. + std::map, int> v2e; + + /// Sets of master edges and face in patchTopo->ncmesh. + std::set masterEdges, masterFaces; + + /// Array form of @a masterEdges. + Array masterEdgeIndex; + + /** @brief Arrays of slave edges or faces, with possible repetitions, ordered + by position within their master entities. */ + std::vector slaveEdges; + std::vector slaveFaces; + + /// Arrays of unique indices in @a slaveEdges, @a slaveFaces. + Array slaveEdgesUnique, slaveFacesUnique; + + /// Maps from slaveEdges/slaveFaces to slaveEdgesUnique/slaveEdgesUnique. + std::map slaveEdgesToUnique, slaveFacesToUnique; + + /// Maps from masterEdges/masterFaces to their indices in an ordered list. + std::map masterEdgeToId, masterFaceToId; + + /// Master edge and face data for a nonconforming patch topology. + std::vector masterEdgeInfo; + std::vector masterFaceInfo; + + /** @brief Get the DOF (dof = true) or vertex (dof = false) offset for the + edge with index @a edge plus @a increment. */ + int GetEdgeOffset(bool dof, int edge, int increment) const; + + /** @brief Get the DOF (dof = true) or vertex (dof = false) offset for the + face with index @a face plus @a increment. */ + int GetFaceOffset(bool dof, int face, int increment) const; + + /// Map from a parent entity vertex index pair to knotvectors on the entity. + std::map, std::array> parentToKV; + + /// Update knot-span indices in @a auxEdges and @a auxFaces on refinement. + void UpdateAuxiliaryKnotSpans(const Array &rf); + + /// For the master edge with index @a mid, set offsets @a os for the number + /// of mesh edges in each subedge (slave or auxiliary edge). + void GetMasterEdgePieceOffsets(int mid, Array &os); + + /// Return the number of mesh edges in auxiliary edge @a aux_edge. + int AuxiliaryEdgeNE(int aux_edge); + + /** @brief Find the permutation @a perm of slave face entities, with entity + perm[i] of the slave face being entity i in the master face ordering. + + @param[in] sf Slave face index in @a patchTopo + @param[in] n1 Number of slave face edges, first master face direction. + @param[in] n2 Number of slave face edges, second master face direction. + @param[in] v0 Bottom-left face vertex with respect to the master face. + @param[in] e1 Local edge index, first direction of the slave face. + @param[in] e2 Local edge index, second direction of the slave face. */ + void GetFaceOrdering(int sf, int n1, int n2, int v0, int e1, int e2, + Array &perm) const; + + /// Find additional slave and auxiliary faces after ProcessVertexToKnot3D. + void FindAdditionalFacesSA( + std::map, int> &v2f, + std::set &addParentFaces, + std::vector &facePairs); + + /// Helper function for @a GenerateOffsets(). + void ProcessFacePairs(int start, int midStart, + const std::vector> &parentSize, + std::vector &parentVerts, + const std::vector &facePairs); + + /// Helper function for @a GenerateOffsets(). + void ProcessVertexToKnot2D(const VertexToKnotSpan &v2k, + std::set &reversedParents, + std::vector &edgePairs); + + /// Helper function for @a GenerateOffsets(). + void ProcessVertexToKnot3D(const VertexToKnotSpan &v2k, + const std::map, int> &v2f, + std::vector> &parentSize, + std::vector &edgePairs, + std::vector &facePairs, + std::vector &parentFaces, + std::vector &parentVerts); + + /// Helper function for @a GenerateOffsets(). + void SetDofToPatch() override; + + /// Helper functions for @a PropagateFactorsForKV(). + void GetAuxFaceToPatchTable(Array2D &auxface2patch); + void GetSlaveFaceToPatchTable(Array2D &sface2patch); + + /// Helper function for @a UniformRefinement(). + void Refine(bool coarsened, const Array *rf = nullptr); + + /// Get the two endpoints of the auxiliary edge with index @a auxEdge. + void GetAuxEdgeVertices(int auxEdge, Array &verts) const; + + /// Get the four vertices of the auxiliary face with index @a auxFace. + void GetAuxFaceVertices(int auxFace, Array &verts) const; + + /// Get the four edges of the auxiliary face with index @a auxFace. + void GetAuxFaceEdges(int auxFace, Array &edges) const; + + /// Helper function for @a SetPatchFactors(). + void SlaveEdgeToParent(int se, int parent, const Array &os, + const std::vector &parentVerts, + Array &edges); + + /// Helper function for @a FindAdditionalFacesSA(). + void GetMasterEdgeEntities(int edge, Array &edgeV, Array &edgeE, + Array &edgeVki); + + /// Helper function for @a Refine(). + void UpdateCoarseKVF(); + + /** @brief Read the control points for coarse patches. + + This is useful for a mesh with a nonconforming patch topology, when + non-nested refinement is done. In such cases, knot insertion is done on + coarse structured patches with a single element. */ + void ReadCoarsePatchCP(std::istream &input) override; + + /// Print control points for coarse patches @a patchCP. + void PrintCoarsePatches(std::ostream &os) override; + + std::vector> auxef; /// Auxiliary edge refinement factors +}; + +} + +#endif diff --git a/mesh/nurbs.cpp b/mesh/nurbs.cpp index 7ec5745152..576e6d6b5b 100644 --- a/mesh/nurbs.cpp +++ b/mesh/nurbs.cpp @@ -137,13 +137,13 @@ KnotVector *KnotVector::DegreeElevate(int t) const return newkv; } -void KnotVector::UniformRefinement(Vector &newknots, int rf) const +void KnotVector::UniformRefinement(Vector &new_knots, int rf) const { MFEM_VERIFY(rf > 1, "Refinement factor must be at least 2."); const real_t h = 1.0 / ((real_t) rf); - newknots.SetSize(NumOfElements * (rf - 1)); + new_knots.SetSize(NumOfElements * (rf - 1)); int j = 0; for (int i = 0; i < knot.Size()-1; i++) { @@ -151,7 +151,7 @@ void KnotVector::UniformRefinement(Vector &newknots, int rf) const { for (int m = 1; m < rf; ++m) { - newknots(j) = ((1.0 - (m * h)) * knot(i)) + (m * h * knot(i+1)); + new_knots(j) = ((1.0 - (m * h)) * knot(i)) + (m * h * knot(i+1)); j++; } } @@ -190,6 +190,7 @@ Vector KnotVector::GetFineKnots(const int cf) const int fcnt = 0; int i = Order; real_t kprev = knot(Order); + int ifine0 = 0; for (int c=0; c mlt(fine.Size()); + mlt = 1; + + for (int j=ifine0+1, ifine=0; j 1, "Refinement factor must be at least 2."); @@ -222,19 +253,20 @@ void KnotVector::Refinement(Vector &newknots, int rf) const { spacing->ScaleParameters(1.0 / ((real_t) rf)); spacing->SetSize(rf * NumOfElements); + Vector s; spacing->EvalAll(s); - newknots.SetSize((rf - 1) * NumOfElements); + new_knots.SetSize(s.Size() - NumOfElements); const real_t k0 = knot(0); - const real_t k1 = knot(knot.Size()-1); + const real_t k1 = knot(knot.Size() - 1); Array span0(NumOfElements + 1); span0[0] = 0; int j = 1; - for (int i = 0; i < knot.Size()-1; i++) + for (int i = 0; i < knot.Size() - 1; i++) { if (knot(i) != knot(i+1)) { @@ -243,10 +275,12 @@ void KnotVector::Refinement(Vector &newknots, int rf) const } } - MFEM_VERIFY(j == NumOfElements + 1, "bug"); + MFEM_VERIFY(j == NumOfElements + 1, "Incorrect number of knot spans"); real_t s0 = 0.0; + int os = 0; + int os1 = 0; for (int i=0; iFlip(); } } void KnotVector::Print(std::ostream &os) const @@ -537,7 +575,7 @@ void KnotVector::FindMaxima(Array &ks, Vector &xi, Vector &u) const xi.SetSize(GetNCP()); u.SetSize(GetNCP()); ks.SetSize(GetNCP()); - for (int j = 0; j Size() == 2 * (Order + 1), ""); + for (int i=0; iGetElements(); + if (spacing) + { + kvc->spacing = spacing->Clone(); + kvc->spacing->FullyCoarsen(); + } + + return kvc; +} + void NURBSPatch::init(int dim) { MFEM_ASSERT(dim > 1, "NURBS patch dimension (including weight) must be " @@ -1051,24 +1109,88 @@ int NURBSPatch::SetLoopDirection(int dir) return -1; } -void NURBSPatch::UniformRefinement(Array const& rf) +void NURBSPatch::UniformRefinement(Array const& rf, int multiplicity) { - Vector newknots; + Vector new_knots; for (int dir = 0; dir < kv.Size(); dir++) { if (rf[dir] != 1) { - kv[dir]->Refinement(newknots, rf[dir]); - KnotInsert(dir, newknots); + kv[dir]->Refinement(new_knots, rf[dir]); + for (int i=0; i> &rf, + bool coarsened, int multiplicity) +{ + Vector new_knots; + for (int dir = 0; dir < kv.Size(); dir++) + { + if (coarsened) + { + const int f = rf[dir].Sum(); + if (f == 1) { continue; } + kv[dir]->Refinement(new_knots, f); + } + else + { + MFEM_VERIFY(rf[dir].IsConstant(), ""); + if (rf[dir][0] == 1) { continue; } + kv[dir]->Refinement(new_knots, rf[dir][0]); + } + + for (int i=0; i rf_array(kv.Size()); rf_array = rf; - UniformRefinement(rf_array); + UniformRefinement(rf_array, multiplicity); +} + +void NURBSPatch::UpdateSpacingPartitions(const Array &pkv) +{ + MFEM_VERIFY(pkv.Size() == kv.Size(), ""); + + for (int dir = 0; dir < kv.Size(); dir++) + { + if (kv[dir]->spacing && pkv[dir]->spacing) + { + PiecewiseSpacingFunction *pws = dynamic_cast + (kv[dir]->spacing.get()); + const PiecewiseSpacingFunction *upws = + dynamic_cast(pkv[dir]->spacing.get()); + + MFEM_VERIFY((pws == nullptr) == (upws == nullptr), ""); + + if (pws) + { + Array s0 = pws->RelativePieceSizes(); + Array s1 = upws->RelativePieceSizes(); + MFEM_ASSERT(s0.Size() == s1.Size(), ""); + + Array rf(s0.Size()); + for (int i=0; iScalePartition(rf, false); + } + } + } } void NURBSPatch::Coarsen(Array const& cf, real_t tol) @@ -1350,8 +1472,8 @@ int NURBSPatch::KnotRemove(int dir, real_t knot, int ntimes, real_t tol) while (j - i > t) { // Compute new control points for one removal step - const real_t a_i = (knot - oldkv[i]) / (oldkv[i+p+1] - oldkv[i]); - const real_t a_j = (knot - oldkv[j]) / (oldkv[j+p+1] - oldkv[j]); + const real_t a_i = (knot - oldkv[i]) / (oldkv[i+p+1+t] - oldkv[i]); + const real_t a_j = (knot - oldkv[j-t]) / (oldkv[j+p+1] - oldkv[j-t]); for (int ll = 0; ll < size; ll++) { @@ -1377,9 +1499,9 @@ int NURBSPatch::KnotRemove(int dir, real_t knot, int ntimes, real_t tol) } else { - const real_t a_i = (knot - oldkv[i]) / (oldkv[i+p+1] - oldkv[i]); + const real_t a_i = (knot - oldkv[i]) / (oldkv[i+p+1+t] - oldkv[i]); for (int ll = 0; ll < size; ll++) - diff[ll] = oldp.slice(i,ll) - (a_i * temp(ii+1, ll)) + diff[ll] = oldp.slice(i,ll) - (a_i * temp(ii+t+1, ll)) - ((1.0 - a_i) * temp(ii-1, ll)); } @@ -1461,11 +1583,11 @@ int NURBSPatch::KnotRemove(int dir, real_t knot, int ntimes, real_t tol) newkv.spacing = oldkv.spacing; newkv.coarse = oldkv.coarse; - for (int k = 0; k < id - ntimes + 1; k++) + for (int k = 0; k < r - ntimes + 1; k++) { newkv[k] = oldkv[k]; } - for (int k = id + 1; k < oldkv.Size(); k++) + for (int k = r + 1; k < oldkv.Size(); k++) { newkv[k - ntimes] = oldkv[k]; } @@ -2033,10 +2155,55 @@ NURBSPatch *Revolve3D(NURBSPatch &patch, real_t n[], real_t ang, int times) void NURBSPatch::SetKnotVectorsCoarse(bool c) { - for (int i=0; icoarse = c; } +} + +void NURBSPatch::FullyCoarsen(const Array2D & cp, int ncp1D) +{ + // Remove interior knots + Array kvc(kv.Size()); + for (int dir = 0; dir < kv.Size(); dir++) { - kv[i]->coarse = c; + kvc[dir] = kv[dir]->FullyCoarsen(); } + + // Copy CP + NURBSPatch *newpatch = new NURBSPatch(kvc, Dim); + NURBSPatch &newp = *newpatch; + + if (Dim == 4) // 3D + { + for (int i=0; iLoadPatchTopo(input, edge_to_ukv); + + Load(input, spacing); +} + +void NURBSExtension::Load(std::istream &input, bool spacing) +{ own_topo = true; CheckPatches(); @@ -2119,9 +2297,39 @@ NURBSExtension::NURBSExtension(std::istream &input, bool spacing) if (spacing) // Read spacing formulas for knotvectors { - input >> ws >> ident; // 'spacing' + input >> ws >> ident; // 'spacing' or 'refinements' + + if (ident == "refinements") + { + ref_factors.SetSize(Dimension()); + for (int i=0; i> ref_factors[i]; + } + + input >> ws >> ident; // 'spacing' + } + + if (ident == "knotvector_refinements") + { + kvf.resize(NumOfKnotVectors); + for (int i=0; i> nf; + kvf[i].SetSize(nf); + for (int j=0; j> kvf[i][j]; + } + } + + input >> ws >> ident; // 'spacing' + } + MFEM_VERIFY(ident == "spacing", "Spacing formula section missing from NURBS mesh file"); + int numSpacing = 0; input >> numSpacing; for (int j = 0; j < numSpacing; j++) @@ -2543,8 +2751,23 @@ void NURBSExtension::Print(std::ostream &os, const std::string &comments) const } } - const int version = kvSpacing.Size() > 0 ? 11 : 10; // v1.0 or v1.1 - patchTopo->PrintTopo(os, edge_to_ukv, version, comments); + bool writeSpacing = false; + bool writeRefinements = false; + if (patchTopo->ncmesh) + { + // Writing MFEM NURBS NC-patch mesh v1.0 + patchTopo->ncmesh->Print(os, comments, true); + patchTopo->PrintTopoEdges(os, edge_to_ukv, true); + writeSpacing = true; + writeRefinements = true; + } + else + { + const int version = kvSpacing.Size() > 0 ? 11 : 10; // v1.0 or v1.1 + if (version == 11) { writeSpacing = true; } + patchTopo->PrintTopo(os, edge_to_ukv, version, comments); + } + if (patches.Size() == 0) { os << "\nknotvectors\n" << NumOfKnotVectors << '\n'; @@ -2553,7 +2776,44 @@ void NURBSExtension::Print(std::ostream &os, const std::string &comments) const knotVectors[i]->Print(os); } - if (kvSpacing.Size() > 0) + if (writeRefinements && ref_factors.Size() > 0) + { + os << "\nrefinements\n"; + for (int i=0; i 0) + { + MFEM_VERIFY(kvf.size() == (size_t) NumOfKnotVectors, ""); + os << "\nknotvector_refinements\n"; + for (size_t i=0; i 0) + { + os << kvf_coarse[i].Size(); + for (int j=0; j edges; - Array oedge; + Array edges, oedge; for (int p = 0; p < GetNP(); p++) { @@ -3074,25 +3333,26 @@ void NURBSExtension::CheckKVDirection(int p, Array &kvdir) { // First side patchTopo->GetEdgeVertices(edges[i], edgevert); + const int ks = KnotSign(edges[i]); if (edgevert[0] == patchvert[0] && edgevert[1] == patchvert[1]) { - kvdir[0] = 1; + kvdir[0] = ks; } if (edgevert[0] == patchvert[1] && edgevert[1] == patchvert[0]) { - kvdir[0] = -1; + kvdir[0] = -ks; } // Second side - if (edgevert[0] == patchvert[1] && edgevert[1] == patchvert[2]) + if (edgevert[0] == patchvert[0] && edgevert[1] == patchvert[3]) { - kvdir[1] = 1; + kvdir[1] = ks; } - if (edgevert[0] == patchvert[2] && edgevert[1] == patchvert[1]) + if (edgevert[0] == patchvert[3] && edgevert[1] == patchvert[0]) { - kvdir[1] = -1; + kvdir[1] = -ks; } } @@ -3102,15 +3362,16 @@ void NURBSExtension::CheckKVDirection(int p, Array &kvdir) for (int i = 0; i < edges.Size(); i++) { patchTopo->GetEdgeVertices(edges[i], edgevert); + const int ks = KnotSign(edges[i]); if (edgevert[0] == patchvert[0] && edgevert[1] == patchvert[4]) { - kvdir[2] = 1; + kvdir[2] = ks; } if (edgevert[0] == patchvert[4] && edgevert[1] == patchvert[0]) { - kvdir[2] = -1; + kvdir[2] = -ks; } } } @@ -3158,10 +3419,8 @@ void NURBSExtension::CreateComprehensiveKV() // Indices in unique and comprehensive sets of the KnotVector int iun = edges[e[d]]; int icomp = Dimension()*p+d; - knotVectorsCompr[icomp] = new KnotVector(*(KnotVec(iun))); - - if (kvdir[d] == -1) {knotVectorsCompr[icomp]->Flip();} + if (kvdir[d] == -1) { knotVectorsCompr[icomp]->Flip(); } } } @@ -3200,10 +3459,10 @@ void NURBSExtension::UpdateUniqueKV() patchTopo->GetElementEdges(p, edges, orient); CheckKVDirection(p, kvdir); - for ( int d = 0; d < Dimension(); d++) + for (int d = 0; d < Dimension(); d++) { bool flip = false; - if (kvdir[d] == -1) {flip = true;} + if (kvdir[d] == -1) { flip = true; } // Indices in unique and comprehensive sets of the KnotVector int iun = edges[e[d]]; @@ -3214,7 +3473,10 @@ void NURBSExtension::UpdateUniqueKV() int o2 = knotVectorsCompr[icomp]->GetOrder(); int diffo = abs(o1 - o2); - if (diffo) + int ne1 = KnotVec(iun)->GetNE(); + int ne2 = knotVectorsCompr[icomp]->GetNE(); + + if (diffo || ne1 != ne2) { // Update reduced set of knotvectors *(KnotVec(iun)) = *(knotVectorsCompr[icomp]); @@ -3339,8 +3601,6 @@ void NURBSExtension::GetPatchKnotVectors(int p, Array &kv) void NURBSExtension::GetPatchKnotVectors(int p, Array &kv) const { - Array edges, orient; - kv.SetSize(Dimension()); if (Dimension() == 1) @@ -3427,14 +3687,13 @@ void NURBSExtension::SetOrdersFromKnotVectors() void NURBSExtension::GenerateOffsets() { - int nv = patchTopo->GetNV(); - int ne = patchTopo->GetNEdges(); - int nf = patchTopo->GetNFaces(); - int np = patchTopo->GetNE(); - int meshCounter, spaceCounter, dim = Dimension(); + const int nv = patchTopo->GetNV(); + const int ne = patchTopo->GetNEdges(); + const int nf = patchTopo->GetNFaces(); + const int np = patchTopo->GetNE(); + int meshCounter, spaceCounter; - Array edges; - Array orient; + Array edges, orient; v_meshOffsets.SetSize(nv); e_meshOffsets.SetSize(ne); @@ -3480,6 +3739,17 @@ void NURBSExtension::GenerateOffsets() } // Get patch offsets + GetPatchOffsets(meshCounter, spaceCounter); + + NumOfVertices = meshCounter; + NumOfDofs = spaceCounter; +} + +void NURBSExtension::GetPatchOffsets(int &meshCounter, int &spaceCounter) +{ + const int np = patchTopo->GetNE(); + const int dim = Dimension(); + Array edges, orient; for (int p = 0; p < np; p++) { p_meshOffsets[p] = meshCounter; @@ -3513,8 +3783,6 @@ void NURBSExtension::GenerateOffsets() (KnotVec(edges[8])->GetNCP() - 2); } } - NumOfVertices = meshCounter; - NumOfDofs = spaceCounter; } void NURBSExtension::CountElements() @@ -4498,8 +4766,7 @@ NURBSExtension* NURBSExtension::GetCurlExtension(int component) return new NURBSExtension(this, newOrders, Mode::H_CURL); } - -void NURBSExtension::UniformRefinement(Array const& rf) +void NURBSExtension::UniformRefinement(const Array &rf) { for (int p = 0; p < patches.Size(); p++) { @@ -4514,7 +4781,7 @@ void NURBSExtension::UniformRefinement(int rf) UniformRefinement(rf_array); } -void NURBSExtension::Coarsen(Array const& cf, real_t tol) +void NURBSExtension::Coarsen(const Array &cf, real_t tol) { // First, mark all knot vectors on all patches as not coarse. This prevents // coarsening the same knot vector twice. @@ -4527,6 +4794,45 @@ void NURBSExtension::Coarsen(Array const& cf, real_t tol) { patches[p]->Coarsen(cf, tol); } + + if (ref_factors.Size() > 0) + { + MFEM_VERIFY(cf.Size() == ref_factors.Size(), ""); + for (int i=0; iSetKnotVectorsCoarse(false); + } + + const int maxOrder = mOrders.Max(); + + // For degree maxOrder, there are 2*(maxOrder + 1) knots for a single element, + // and the number of control points in each dimension is + // 2*(maxOrder + 1) - maxOrder - 1 + const int ncp1D = maxOrder + 1; + const int ncp = pow(ncp1D, Dimension()); + + for (int p = 0; p < patches.Size(); p++) + { + if (p < num_structured_patches) + { + // Use data from patchCP + Array2D pcp(ncp, Dimension()); + for (int i=0; iFullyCoarsen(pcp, ncp1D); + } + } } void NURBSExtension::Coarsen(int cf, real_t tol) @@ -4552,11 +4858,21 @@ void NURBSExtension::GetCoarseningFactors(Array & f) const MFEM_VERIFY(f.Size() == pf.Size(), ""); for (int i=0; i kv(2); NURBSPatchMap p2g(this); + const bool d2p = dof2patch.Size() > 0; + weights.SetSize(GetNDof()); for (int p = 0; p < GetNP(); p++) { @@ -4893,6 +5209,8 @@ void NURBSExtension::Set2DSolutionVector(Vector &coords, int vdim) for (int i = 0; i < kv[0]->GetNCP(); i++) { const int l = p2g(i,j); + if (d2p && dof2patch[l] >= 0 && dof2patch[l] != p) { continue; } + for (int d = 0; d < vdim; d++) { coords(l*vdim + d) = patch(i,j,d)/patch(i,j,vdim); @@ -4909,6 +5227,8 @@ void NURBSExtension::Set3DSolutionVector(Vector &coords, int vdim) Array kv(3); NURBSPatchMap p2g(this); + const bool d2p = dof2patch.Size() > 0; + weights.SetSize(GetNDof()); for (int p = 0; p < GetNP(); p++) { @@ -4923,6 +5243,8 @@ void NURBSExtension::Set3DSolutionVector(Vector &coords, int vdim) for (int i = 0; i < kv[0]->GetNCP(); i++) { const int l = p2g(i,j,k); + if (d2p && dof2patch[l] >= 0 && dof2patch[l] != p) { continue; } + for (int d = 0; d < vdim; d++) { coords(l*vdim + d) = patch(i,j,k,d)/patch(i,j,k,vdim); @@ -4987,6 +5309,90 @@ const Array& NURBSExtension::GetPatchBdrElements(int patch) return patch_to_bel[patch]; } +void NURBSExtension::GetVertexDofs(int vertex, Array &dofs) const +{ + MFEM_ASSERT(vertex < v_spaceOffsets.Size(), ""); + + const int os = v_spaceOffsets[vertex]; + const int os1 = vertex + 1 == v_spaceOffsets.Size() ? e_spaceOffsets[0] : + v_spaceOffsets[vertex + 1]; + + dofs.SetSize(0); + dofs.Reserve(os1 - os); + + for (int i=os; i &dofs) const +{ + MFEM_ASSERT(edge < e_spaceOffsets.Size(), ""); + + const int os = e_spaceOffsets[edge]; + const int os_upper = f_spaceOffsets.Size() > 0 ? f_spaceOffsets[0] : + p_spaceOffsets[0]; + const int os1 = edge + 1 == e_spaceOffsets.Size() ? os_upper : + v_spaceOffsets[edge + 1]; + + dofs.SetSize(0); + // Reserve 2 for the two vertices and os1 - os for the interior edge DOFs. + dofs.Reserve(2 + os1 - os); + + // First get the DOFs for the vertices of the edge. + + Array vert; + patchTopo->GetEdgeVertices(edge, vert); + + for (auto v : vert) + { + Array vdofs; + GetVertexDofs(v, vdofs); + dofs.Append(vdofs); + } + + // Now get the interior edge DOFs. + for (int i=os; i &vertices) const +{ + MFEM_ABORT("VertexPairToEdge is supported only in NCNURBSExtension"); + return -1; +} + +void NURBSExtension::GetMasterEdgeDofs(bool dof, int me, Array &dofs) const +{ + MFEM_ABORT("GetMasterEdgeDofs is supported only in NCNURBSExtension"); +} + +void NURBSExtension::GetMasterFaceDofs(bool dof, int mf, + Array2D &dofs) const +{ + MFEM_ABORT("GetMasterFaceDofs is supported only in NCNURBSExtension"); +} + +void NURBSExtension::RefineWithKVFactors(int rf, + const std::string &kvf_filename, + bool coarsened) +{ + MFEM_ABORT("RefineWithKVFactors is supported only in NCNURBSExtension"); +} + NURBSPatch::NURBSPatch(const KnotVector *kv0, const KnotVector *kv1, int dim_, const real_t* control_points) { @@ -5391,6 +5797,7 @@ void ParNURBSExtension::BuildGroups(const int *partition, IntegerSet group; Transpose(elem_dof, dof_proc); // dof_proc is dof_elem + // convert elements to processors for (int i = 0; i < dof_proc.Size_of_connections(); i++) { @@ -5465,7 +5872,6 @@ void NURBSPatchMap::GetBdrPatchKnotVectors(int p, const KnotVector *kv[], kv[0] = Ext->KnotVec(edges[0], oedge[0], &okv[0]); kv[1] = Ext->KnotVec(edges[1], oedge[1], &okv[1]); } - } void NURBSPatchMap::SetPatchVertexMap(int p, const KnotVector *kv[]) @@ -5482,6 +5888,7 @@ void NURBSPatchMap::SetPatchVertexMap(int p, const KnotVector *kv[]) if (Ext->Dimension() >= 2) { J = kv[1]->GetNE() - 1; + SetMasterEdges(false, kv); for (int i = 0; i < edges.Size(); i++) { edges[i] = Ext->e_meshOffsets[edges[i]]; @@ -5490,7 +5897,7 @@ void NURBSPatchMap::SetPatchVertexMap(int p, const KnotVector *kv[]) if (Ext->Dimension() == 3) { K = kv[2]->GetNE() - 1; - + SetMasterFaces(false); for (int i = 0; i < faces.Size(); i++) { faces[i] = Ext->f_meshOffsets[faces[i]]; @@ -5513,15 +5920,33 @@ void NURBSPatchMap::SetPatchDofMap(int p, const KnotVector *kv[]) if (Ext->Dimension() >= 2) { J = kv[1]->GetNCP() - 2; - for (int i = 0; i < edges.Size(); i++) + SetMasterEdges(true); + + if (Ext->NonconformingPatches() && Ext->patchTopo->ncmesh + && Ext->patchTopo->ncmesh->GetVertexToKnotSpan().Size() > 0) { - edges[i] = Ext->e_spaceOffsets[edges[i]]; + for (int i = 0; i < edges.Size(); i++) + { + // Find the patchTopo->ncmesh edge corresponding to edges[i]. + Array vert; + Ext->patchTopo->GetEdgeVertices(edges[i], vert); + const std::pair vpair(vert[0], vert[1]); + const int ncedge = Ext->VertexPairToEdge(vpair); + edges[i] = Ext->e_spaceOffsets[ncedge]; + } + } + else + { + for (int i = 0; i < edges.Size(); i++) + { + edges[i] = Ext->e_spaceOffsets[edges[i]]; + } } } if (Ext->Dimension() == 3) { K = kv[2]->GetNCP() - 2; - + SetMasterFaces(true); for (int i = 0; i < faces.Size(); i++) { faces[i] = Ext->f_spaceOffsets[faces[i]]; @@ -5549,12 +5974,15 @@ void NURBSPatchMap::SetBdrPatchVertexMap(int p, const KnotVector *kv[], { I = kv[0]->GetNE() - 1; pOffset = Ext->e_meshOffsets[edges[0]]; + SetMasterEdges(false); } else if (Ext->Dimension() == 3) { I = kv[0]->GetNE() - 1; J = kv[1]->GetNE() - 1; + SetMasterEdges(false); + SetMasterFaces(false); for (int i = 0; i < edges.Size(); i++) { edges[i] = Ext->e_meshOffsets[edges[i]]; @@ -5581,12 +6009,15 @@ void NURBSPatchMap::SetBdrPatchDofMap(int p, const KnotVector *kv[], int *okv) { I = kv[0]->GetNCP() - 2; pOffset = Ext->e_spaceOffsets[edges[0]]; + + SetMasterEdges(true); } else if (Ext->Dimension() == 3) { I = kv[0]->GetNCP() - 2; J = kv[1]->GetNCP() - 2; + SetMasterEdges(true); for (int i = 0; i < edges.Size(); i++) { edges[i] = Ext->e_spaceOffsets[edges[i]]; diff --git a/mesh/nurbs.hpp b/mesh/nurbs.hpp index a300fe75e1..ec2c159782 100644 --- a/mesh/nurbs.hpp +++ b/mesh/nurbs.hpp @@ -28,7 +28,6 @@ namespace mfem class GridFunction; - /** @brief A vector of knots in one dimension, with B-spline basis functions of a prescribed order. @@ -158,10 +157,10 @@ public: void Difference(const KnotVector &kv, Vector &diff) const; /// Uniformly refine by factor @a rf, by inserting knots in each span. - void UniformRefinement(Vector &newknots, int rf) const; + void UniformRefinement(Vector &new_knots, int rf) const; /// Refine with refinement factor @a rf. - void Refinement(Vector &newknots, int rf) const; + void Refinement(Vector &new_knots, int rf) const; /** Returns the coarsening factor needed for non-nested nonuniform spacing functions, to result in a single element from which refinement can be @@ -201,11 +200,14 @@ public: /// Const access function to knot @a i. const real_t &operator[](int i) const { return knot(i); } + /// Coarsen to a single element. + KnotVector* FullyCoarsen(); + /// Function to define the distribution of knots for any number of knot spans. std::shared_ptr spacing; - /** Flag to indicate whether the KnotVector has been coarsened, which means - it is ready for non-nested refinement. */ + /** @brief Flag to indicate whether the KnotVector has been coarsened, which + means it is ready for non-nested refinement. */ bool coarse; #ifdef MFEM_USE_LAPACK @@ -255,7 +257,6 @@ protected: Returns the number of variables per knot in flattened structure. */ int SetLoopDirection(int dir); - /** @brief Access function for the effectively 1D flattened net, where @a i is a knot index, and @a j is an index of a variable per knot. */ inline real_t &slice(int i, int j); @@ -319,6 +320,9 @@ public: /// Increase the order in direction @a dir by @a t >= 0. void DegreeElevate(int dir, int t); + /// Increase the order in all directions by @a t >= 0. + void DegreeElevate(int t); + /** @brief Insert any new knots from @a knot in direction @a dir. If the order of @a knot is higher than the current order in direction @a dir, then the order is elevated in that direction to match. */ @@ -348,17 +352,20 @@ public: /// Remove all knots in @a knot once, for each direction. void KnotRemove(Array &knot, real_t tol = 1.0e-12); - void DegreeElevate(int t); - /** @brief Refine with optional refinement factor @a rf. Uniform means refinement is done everywhere by the same factor, although nonuniform spacing functions may be used. @param[in] rf Optional refinement factor. If scalar, the factor is used for all dimensions. If an array, factors can be specified - for each dimension. */ - void UniformRefinement(int rf = 2); - void UniformRefinement(Array const& rf); + for each dimension. + @param[in] multiplicity Optional multiplicity for new knots inserted. */ + void UniformRefinement(int rf = 2, int multiplicity = 1); + void UniformRefinement(const Array &rf, int multiplicity = 1); + + /// Flag @a coarsened indicates whether the patch is a single element. + void UniformRefinement(const std::vector> &rf, + bool coarsened = false, int multiplicity = 1); /** @brief Coarsen with optional coarsening factor @a cf which divides the number of elements in each dimension. Nonuniform spacing functions may be @@ -370,14 +377,20 @@ public: @param[in] tol NURBS geometry deviation tolerance, cf. Algorithm A5.8 of "The NURBS Book", 2nd ed, Piegl and Tiller. */ void Coarsen(int cf = 2, real_t tol = 1.0e-12); - void Coarsen(Array const& cf, real_t tol = 1.0e-12); + void Coarsen(const Array &cf, real_t tol = 1.0e-12); /// Calls KnotVector::GetCoarseningFactor for each direction. - void GetCoarseningFactors(Array & f) const; + void GetCoarseningFactors(Array &f) const; /// Marks the KnotVector in each dimension as coarse. void SetKnotVectorsCoarse(bool c); + /// Coarsen to a single element. + void FullyCoarsen(const Array2D &cp, int ncp1D); + + /// Update piecewise spacing function partitions to match refined @a pkv. + void UpdateSpacingPartitions(const Array &pkv); + /// Return the number of components stored in the NURBSPatch int GetNC() const { return Dim; } @@ -424,7 +437,7 @@ public: /// Rotate the NURBSPatch in 2D or 3D.. /** A rotation of a 2D NURBS-patch requires an angle only. Rotating a 3D NURBS-patch requires a normal as well.*/ - void Rotate(real_t angle, real_t normal[]= NULL); + void Rotate(real_t angle, real_t normal[] = NULL); /// Rotate the NURBSPatch, 2D case. void Rotate2D(real_t angle); @@ -555,6 +568,22 @@ protected: /// Return the unsigned index of the KnotVector for edge @a edge. inline int KnotInd(int edge) const; + /// Return the sign (orientation) of the KnotVector for edge @a edge. + inline int KnotSign(int edge) const; + + bool nonconformingPT = false; /// Whether patchTopo is a nonconforming mesh. + + int num_structured_patches = 0; /// Number of structured patches + + Array3D patchCP; /// Control points for coarse structured patches + + std::vector> kvf, kvf_coarse; /// Knotvector refinement factors + + Array ref_factors; /// Refinement factors in each dimension. + + static constexpr int unsetFactor = 0; /// Unset refinement factor value + + Array dof2patch; /// DOF to owning patch map in @a SetSolutionVector() /// Access function for the KnotVector associated with edge @a edge. /// @note The returned object should NOT be deleted by the caller. @@ -562,9 +591,9 @@ protected: /// Const access function for the KnotVector associated with edge @a edge. /// @note The returned object should NOT be deleted by the caller. inline const KnotVector *KnotVec(int edge) const; - /* brief Const access function for the KnotVector associated with edge - @a edge. The output orientation @a okv is set to @a oedge with sign flipped - if the KnotVector index associated with edge @a edge is negative. */ + /** @brief Const access function for the KnotVector associated with edge + @a edge. The output orientation @a okv is set to @a oedge with sign flipped + if the KnotVector index associated with edge @a edge is negative. */ inline const KnotVector *KnotVec(int edge, int oedge, int *okv) const; /// Throw an error if any patch has an inconsistent edge_to_ukv mapping. @@ -615,7 +644,7 @@ protected: /** @brief Set the mesh and space offsets, and also count the global @a NumOfVertices and the global @a NumOfDofs. */ - void GenerateOffsets(); + virtual void GenerateOffsets(); /// Count the global @a NumOfElements. void CountElements(); @@ -685,16 +714,50 @@ protected: /// Set @a patch_to_bel. void SetPatchToBdrElements(); + /// Load data from file (used by constructor). + void Load(std::istream &input, bool spacing); + + /// Return true if @a edge is a master NC-patch edge. + virtual bool IsMasterEdge(int edge) const { return false; } + + /// Return true if @a face is a master NC-patch face. + virtual bool IsMasterFace(int face) const { return false; } + + /// Given a pair of vertices, return the corresponding edge. + virtual int VertexPairToEdge(const std::pair &vertices) const; + + /** @brief Get the DOFs (dof = true) or vertices (dof = false) for + master edge @a me. */ + virtual void GetMasterEdgeDofs(bool dof, int me, Array &dofs) const; + + /** @brief Get the DOFs (dof = true) or vertices (dof = false) for + master face @a mf. */ + virtual void GetMasterFaceDofs(bool dof, int mf, Array2D &dofs) const; + + /// Helper function for @a GenerateOffsets(). + void GetPatchOffsets(int &meshCounter, int &spaceCounter); + /// Return NURBSPatch object; returned object should NOT be deleted. const NURBSPatch* GetPatch(int patch) const { return patches[patch]; } /// To be used by ParNURBSExtension constructor(s) NURBSExtension() : el_dof(nullptr), bel_dof(nullptr) { } +private: + /// Get the degrees of freedom for the vertex @a vertex in @a dofs. + void GetVertexDofs(int vertex, Array &dofs) const; + + /// Get the degrees of freedom for the edge @a edge in @a dofs. + void GetEdgeDofs(int edge, Array &dofs) const; + + // TODO: does this still need to be virtual? + /// Helper function for @a GenerateOffsets(). + virtual void SetDofToPatch() { }; + public: /// Copy constructor: deep copy NURBSExtension(const NURBSExtension &orig); - /// Read-in a NURBSExtension from a stream @a input.. + /// Read-in a NURBSExtension from a stream @a input. NURBSExtension(std::istream &input, bool spacing=false); /** @brief Create a NURBSExtension with elevated order by repeating the endpoints of the KnotVectors and using uniform weights of 1. */ @@ -850,6 +913,9 @@ public: int GetPatchBdrAttribute(int i) const { return patchTopo->GetBdrAttribute(i); } + /// Return the number of knotvector elements for edge @a edge. + inline int KnotVecNE(int edge) const; + // Load functions /// Load element @a i into @a FE. @@ -885,16 +951,23 @@ public: void DegreeElevate(int rel_degree, int degree = 16); /** @brief Refine with optional refinement factor @a rf. Uniform means - refinement is done everywhere by the same factor, although nonuniform - spacing functions may be used. - */ + refinement is done everywhere by the same factor, although nonuniform + spacing functions may be used. */ void UniformRefinement(int rf = 2); - void UniformRefinement(Array const& rf); + virtual void UniformRefinement(const Array &rf); + + /// Refine with refinement factors loaded for some knotvectors specified in + /// the given file, with default refinement factor @a rf elsewhere. The flag + /// @a coarsened indicates whether each patch is a single element. + virtual void RefineWithKVFactors(int rf, const std::string &kvf_filename, + bool coarsened); + + /// Coarsen with optional coarsening factor @a cf. void Coarsen(int cf = 2, real_t tol = 1.0e-12); void Coarsen(Array const& cf, real_t tol = 1.0e-12); /** @brief Insert knots from @a kv into all KnotVectors in all patches. The - size of @a kv should be the same as @a knotVectors. */ + size of @a kv should be the same as @a knotVectors. */ void KnotInsert(Array &kv); void KnotInsert(Array &kv); @@ -913,8 +986,7 @@ public: /** Calls GetCoarseningFactors for each patch and finds the minimum factor for each direction that ensures refinement will work in the case of non-nested spacing functions. */ - void GetCoarseningFactors(Array & f) const; - + void GetCoarseningFactors(Array &f) const; /// Returns the index of the patch containing element @a elem. int GetElementPatch(int elem) const { return el_to_patch[elem]; } @@ -939,6 +1011,22 @@ public: const Array& GetPatchElements(int patch); /// Return the array of indices of all boundary elements in patch @a patch. const Array& GetPatchBdrElements(int patch); + + /// Return true if the patch topology mesh is nonconforming. + bool NonconformingPatches() const { return nonconformingPT; } + + /// Return a pointer to the NCMesh of a nonconforming patch topology mesh. + NCMesh *GetNCMesh() const { return patchTopo->ncmesh; } + + /// Read the control points for coarse patches. + virtual void ReadCoarsePatchCP(std::istream &input); + + /** @brief Fully coarsen all structured patches, for non-nested refinement of + a mesh with a nonconforming patch topology. */ + void FullyCoarsen(); + + /// Print control points for coarse patches. + virtual void PrintCoarsePatches(std::ostream &os); }; @@ -989,23 +1077,31 @@ public: #endif -/** @brief Mapping for mesh vertices and NURBS space DOFs. */ +/** @brief Mapping for mesh vertices and NURBS space DOFs on a patch. + + This class has two modes, for vertices or DOFs, depending on whether + @a SetPatchVertexMap or @a SetPatchDofMap is called. + */ class NURBSPatchMap { private: /// This object must be associated with exactly one NURBSExtension. const NURBSExtension *Ext; - /// Number of elements in each direction, minus 1. + /// Vertex mode: Number of elements in each direction, minus 1. + /// DOF mode: Number of control points in each direction, minus 2. int I, J, K; - /// Vertex of DOF offset for this patch, among all patches. + /// Vertex/DOF offset for this patch, among all patches. int pOffset; /// Orientation for this boundary patch (0 in the patch case). int opatch; /// Patch topology entities for this patch or boundary patch. Array verts, edges, faces, oedge, oface; + Array edgeMaster, faceMaster; + Array edgeMasterOffset, faceMasterOffset; + Array masterDofs; inline static int F(const int n, const int N) { return (n < 0) ? 0 : ((n >= N) ? 2 : 1); } @@ -1016,6 +1112,27 @@ private: inline static int Or2D(const int n1, const int n2, const int N1, const int N2, const int Or); + inline int EC(const int e, const int n, const int N, const int s=1) const + { + return !edgeMaster[e] ? edges[e] + Or1D(n, N, s*oedge[e]) : + GetMasterEdgeDof(e, Or1D(n, N, s*oedge[e])); + } + + inline int FC(const int f, const int m, const int n, + const int M, const int N) const + { + return !faceMaster[f] ? faces[f] + Or2D(m, n, M, N, oface[f]) : + GetMasterFaceDof(f, Or2D(m, n, M, N, oface[f])); + } + + inline int FCP(const int f, const int m, const int n, + const int M, const int N) const + { + return (faceMaster.Size() == 0 || !faceMaster[f]) ? + pOffset + Or2D(m, n, M, N, opatch) : + GetMasterFaceDof(f, Or2D(m, n, M, N, opatch)); + } + // The following 2 functions also set verts, edges, faces, orientations etc. /// Get the KnotVectors for patch @a p in @a kv. @@ -1024,15 +1141,25 @@ private: orientations output in @a okv. */ void GetBdrPatchKnotVectors(int bp, const KnotVector *kv[], int *okv); + void SetMasterEdges(bool dof, const KnotVector *kv[] = nullptr); + void SetMasterFaces(bool dof); + int GetMasterEdgeDof(const int e, const int i) const; + int GetMasterFaceDof(const int f, const int i) const; + public: /// Constructor for an object associated with NURBSExtension @a ext. NURBSPatchMap(const NURBSExtension *ext) { Ext = ext; } - /// Return the number of elements in the first direction. + /// Vertex mode: Return the number of elements in the first direction. + /// DOF mode: Return the number of control points - 1 in the first direction. inline int nx() const { return I + 1; } - /// Return the number of elements in the second direction (2D or 3D). + + /// Vertex mode: Return the number of elements in the second direction (2D or 3D). + /// DOF mode: Return the number of control points - 1 in the second direction (2D or 3D). inline int ny() const { return J + 1; } - /// Return the number of elements in the third direction (3D). + + /// Vertex mode: Return the number of elements in the third direction (3D). + /// DOF mode: Return the number of control points - 1 in the third direction (3D). inline int nz() const { return K + 1; } /// Set mesh vertex map for patch @a p with KnotVectors @a kv. @@ -1162,8 +1289,13 @@ inline const real_t &NURBSPatch::operator()(int i, int j, int k, int l) const inline int NURBSExtension::KnotInd(int edge) const { - int kv = edge_to_ukv[edge]; - return (kv >= 0) ? kv : (-1-kv); + const int kv = edge_to_ukv[edge]; + return kv >= 0 ? kv : -1 - kv; +} + +inline int NURBSExtension::KnotSign(int edge) const +{ + return edge_to_ukv[edge] >= 0 ? 1 : -1; } inline KnotVector *NURBSExtension::KnotVec(int edge) @@ -1192,12 +1324,15 @@ const } } +inline int NURBSExtension::KnotVecNE(int edge) const +{ + return knotVectors[KnotInd(edge)]->GetNE(); +} // static method inline int NURBSPatchMap::Or2D(const int n1, const int n2, const int N1, const int N2, const int Or) { - // Needs testing switch (Or) { case 0: return n1 + n2*N1; @@ -1221,7 +1356,9 @@ inline int NURBSPatchMap::operator()(const int i) const switch (F(i1, I)) { case 0: return verts[0]; - case 1: return pOffset + Or1D(i1, I, opatch); + case 1: return edgeMaster.Size() > 0 && edgeMaster[0] ? + GetMasterEdgeDof(0, Or1D(i1, I, opatch)) : + pOffset + Or1D(i1, I, opatch); case 2: return verts[1]; } #ifdef MFEM_DEBUG @@ -1236,13 +1373,13 @@ inline int NURBSPatchMap::operator()(const int i, const int j) const switch (3*F(j1, J) + F(i1, I)) { case 0: return verts[0]; - case 1: return edges[0] + Or1D(i1, I, oedge[0]); + case 1: return EC(0, i1, I); case 2: return verts[1]; - case 3: return edges[3] + Or1D(j1, J, -oedge[3]); - case 4: return pOffset + Or2D(i1, j1, I, J, opatch); - case 5: return edges[1] + Or1D(j1, J, oedge[1]); + case 3: return EC(3, j1, J, -1); + case 4: return FCP(0, i1, j1, I, J); + case 5: return EC(1, j1, J); case 6: return verts[3]; - case 7: return edges[2] + Or1D(i1, I, -oedge[2]); + case 7: return EC(2, i1, I, -1); case 8: return verts[2]; } #ifdef MFEM_DEBUG @@ -1254,36 +1391,35 @@ inline int NURBSPatchMap::operator()(const int i, const int j) const inline int NURBSPatchMap::operator()(const int i, const int j, const int k) const { - // Needs testing const int i1 = i - 1, j1 = j - 1, k1 = k - 1; switch (3*(3*F(k1, K) + F(j1, J)) + F(i1, I)) { case 0: return verts[0]; - case 1: return edges[0] + Or1D(i1, I, oedge[0]); + case 1: return EC(0, i1, I); case 2: return verts[1]; - case 3: return edges[3] + Or1D(j1, J, oedge[3]); - case 4: return faces[0] + Or2D(i1, J - 1 - j1, I, J, oface[0]); - case 5: return edges[1] + Or1D(j1, J, oedge[1]); + case 3: return EC(3, j1, J); + case 4: return FC(0, i1, J - 1 - j1, I, J); + case 5: return EC(1, j1, J); case 6: return verts[3]; - case 7: return edges[2] + Or1D(i1, I, oedge[2]); + case 7: return EC(2, i1, I); case 8: return verts[2]; - case 9: return edges[8] + Or1D(k1, K, oedge[8]); - case 10: return faces[1] + Or2D(i1, k1, I, K, oface[1]); - case 11: return edges[9] + Or1D(k1, K, oedge[9]); - case 12: return faces[4] + Or2D(J - 1 - j1, k1, J, K, oface[4]); + case 9: return EC(8, k1, K); + case 10: return FC(1, i1, k1, I, K); + case 11: return EC(9, k1, K); + case 12: return FC(4, J - 1 - j1, k1, J, K); case 13: return pOffset + I*(J*k1 + j1) + i1; - case 14: return faces[2] + Or2D(j1, k1, J, K, oface[2]); - case 15: return edges[11] + Or1D(k1, K, oedge[11]); - case 16: return faces[3] + Or2D(I - 1 - i1, k1, I, K, oface[3]); - case 17: return edges[10] + Or1D(k1, K, oedge[10]); + case 14: return FC(2, j1, k1, J, K); + case 15: return EC(11, k1, K); + case 16: return FC(3, I - 1 - i1, k1, I, K); + case 17: return EC(10, k1, K); case 18: return verts[4]; - case 19: return edges[4] + Or1D(i1, I, oedge[4]); + case 19: return EC(4, i1, I); case 20: return verts[5]; - case 21: return edges[7] + Or1D(j1, J, oedge[7]); - case 22: return faces[5] + Or2D(i1, j1, I, J, oface[5]); - case 23: return edges[5] + Or1D(j1, J, oedge[5]); + case 21: return EC(7, j1, J); + case 22: return FC(5, i1, j1, I, J); + case 23: return EC(5, j1, J); case 24: return verts[7]; - case 25: return edges[6] + Or1D(i1, I, oedge[6]); + case 25: return EC(6, i1, I); case 26: return verts[6]; } #ifdef MFEM_DEBUG diff --git a/mesh/pmesh.cpp b/mesh/pmesh.cpp index 69dfd298cc..10aa621950 100644 --- a/mesh/pmesh.cpp +++ b/mesh/pmesh.cpp @@ -1590,21 +1590,27 @@ void ParMesh::DistributeAttributes(Array &attr) delete [] glb_attr_marker; } -void ParMesh::SetAttributes() +void ParMesh::SetAttributes(bool elem_attrs_changed, bool bdr_attrs_changed) { // Determine the attributes occurring in local interior and boundary elements - Mesh::SetAttributes(); + Mesh::SetAttributes(elem_attrs_changed, bdr_attrs_changed); - DistributeAttributes(bdr_attributes); - if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0) + if (bdr_attrs_changed) { - MFEM_WARNING("Non-positive boundary element attributes found!"); + DistributeAttributes(bdr_attributes); + if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0) + { + MFEM_WARNING("Non-positive boundary element attributes found!"); + } } - DistributeAttributes(attributes); - if (attributes.Size() > 0 && attributes[0] <= 0) + if (elem_attrs_changed) { - MFEM_WARNING("Non-positive element attributes found!"); + DistributeAttributes(attributes); + if (attributes.Size() > 0 && attributes[0] <= 0) + { + MFEM_WARNING("Non-positive element attributes found!"); + } } } @@ -3891,6 +3897,13 @@ void ParMesh::LocalRefinement(const Array &marked_el, int type) #endif } +bool ParMesh::AnisotropicConflict(const Array &refinements, + std::set &conflicts) const +{ + MFEM_VERIFY(pncmesh, "AnisotropicConflict should be called only for NCMesh"); + return pncmesh->AnisotropicConflict(refinements, conflicts); +} + void ParMesh::NonconformingRefinement(const Array &refinements, int nc_limit) { @@ -4588,6 +4601,14 @@ void ParMesh::NURBSUniformRefinement(const Array &rf, real_t tol) } } +void ParMesh::RefineNURBSWithKVFactors(int rf, const std::string &kvf) +{ + if (MyRank == 0) + { + mfem::out << "\nRefineNURBSWithKVFactors : Not supported yet!\n"; + } +} + void ParMesh::PrintXG(std::ostream &os) const { MFEM_ASSERT(Dim == spaceDim, "2D manifolds not supported"); @@ -6404,7 +6425,7 @@ void ParMesh::PrintVTU(std::string pathname, VTKFormat format, bool high_order_output, int compression_level, - bool bdr) + bool bdr_elements) { int pad_digits_rank = 6; DataCollection::create_directory(pathname, this, MyRank); @@ -6464,7 +6485,8 @@ void ParMesh::PrintVTU(std::string pathname, std::string vtu_fname = pathname + "/" + fname + ".proc" + to_padded_string(MyRank, pad_digits_rank); - Mesh::PrintVTU(vtu_fname, format, high_order_output, compression_level, bdr); + Mesh::PrintVTU(vtu_fname, format, high_order_output, compression_level, + bdr_elements); } int ParMesh::FindPoints(DenseMatrix& point_mat, Array& elem_id, diff --git a/mesh/pmesh.hpp b/mesh/pmesh.hpp index bad6aafbe7..04f6316536 100644 --- a/mesh/pmesh.hpp +++ b/mesh/pmesh.hpp @@ -191,6 +191,8 @@ protected: void NURBSUniformRefinement(int rf = 2, real_t tol=1.0e-12) override; void NURBSUniformRefinement(const Array &rf, real_t tol=1.e-12) override; + void RefineNURBSWithKVFactors(int rf, const std::string &kvf) override; + /// This function is not public anymore. Use GeneralRefinement instead. void LocalRefinement(const Array &marked_el, int type = 3) override; @@ -394,7 +396,8 @@ public: void Finalize(bool refine = false, bool fix_orientation = false) override; - void SetAttributes() override; + void SetAttributes(bool elem_attrs_changed = true, + bool bdr_attrs_changed = true) override; /// Checks if any rank in the mesh has boundary elements bool HasBoundaryElements() const override; @@ -787,7 +790,7 @@ public: VTKFormat format=VTKFormat::ASCII, bool high_order_output=false, int compression_level=0, - bool bdr=false) override; + bool bdr_elements=false) override; /// Parallel version of Mesh::Load(). void Load(std::istream &input, int generate_edges = 0, @@ -814,6 +817,22 @@ public: /// Debugging method void PrintSharedEntities(const std::string &fname_prefix) const; + /** @brief Return true if the input array of refinements to be performed would + result in conflicting anisotropic directions on a face. Indices of + @a refinements entries are contained in @a conflicts, for marked elements + neighboring a face with a conflict. + + The return value is globally MPI-reduced (true if any MPI process has a + conflict), whereas @a conflicts contains local indices of conflicting + entries of @a refinements. Conflicts are defined as anisotropic + refinements in different directions on a face shared by two elements. + Conflicts are checked for the mesh that would result from the input + refinements. If there are no conflicts, then the refinements can be + performed without forced refinements. This function is supported only for + 3D meshes with all hexahedral elements. */ + bool AnisotropicConflict(const Array &refinements, + std::set &conflicts) const; + virtual ~ParMesh(); }; diff --git a/mesh/pncmesh.cpp b/mesh/pncmesh.cpp index ba372687da..323b885ce3 100644 --- a/mesh/pncmesh.cpp +++ b/mesh/pncmesh.cpp @@ -1504,6 +1504,507 @@ void ParNCMesh::Prune() Update(); } +bool ParNCMesh::AnisotropicConflict(const Array &refinements, + std::set &conflicts) +{ + if (Dim < 3 || NRanks == 1) { return false; } + + for (int i = 0; i < refinements.Size() && Iso; i++) + { + const Refinement &ref = refinements[i]; + if (ref.GetType() != Refinement::XYZ) + { + Iso = false; + } + } + + // Reduce the Iso flag over all MPI ranks. + bool globalIso = false; + MPI_Allreduce(&Iso, &globalIso, 1, MFEM_MPI_CXX_BOOL, MPI_LAND, MyComm); + + if (globalIso) { return false; } + + // In the 3D parallel anisotropic case, check for conflicts on faces. + NeighborRefinementMessage::Map send_ref; + + // Create refinement messages to all neighbors (NOTE: some may be empty). + Array neighbors; + NeighborProcessors(neighbors); + for (int i = 0; i < neighbors.Size(); i++) + { + send_ref[neighbors[i]].SetNCMesh(this); + } + + // Populate messages: all refinements that occur next to the processor + // boundary need to be sent to the adjoining neighbors so they can keep + // their ghost layer up to date. + Array ranks; + ranks.Reserve(64); + for (int i = 0; i < refinements.Size(); i++) + { + const Refinement &ref = refinements[i]; + MFEM_ASSERT(ref.index < NElements, ""); + const int elem = leaf_elements[ref.index]; + ElementNeighborProcessors(elem, ranks); + for (int j = 0; j < ranks.Size(); j++) + { + send_ref[ranks[j]].AddRefinement(elem, ref.GetType()); + } + } + + // Send the messages (overlap with local refinements) + NeighborRefinementMessage::IsendAll(send_ref, MyComm); + + // Note that ghost refinements are not looked up using elemToRef. Local + // refinements are recorded first in elemToRef, and ghosts only need to be + // compared to local refinements. There is no need for ghost-to-ghost + // comparisons. + std::map elemToRef; // Only for local refinements, not ghosts. + for (int i = 0; i < refinements.Size(); i++) + { + elemToRef[leaf_elements[refinements[i].index]] = i; + } + + // Check local refinements + for (int i = 0; i < refinements.Size(); i++) + { + const Refinement &ref = refinements[i]; + CheckRefinement(leaf_elements[ref.index], ref.GetType(), refinements, + elemToRef, conflicts); + } + + // Receive (ghost layer) refinements from all neighbors + for (int j = 0; j < neighbors.Size(); j++) + { + int rank, size; + NeighborRefinementMessage::Probe(rank, size, MyComm); + + NeighborRefinementMessage msg; + msg.SetNCMesh(this); + msg.Recv(rank, size, MyComm); + + // check the ghost refinements + for (int i = 0; i < msg.Size(); i++) + { + CheckRefinement(msg.elements[i], msg.values[i], refinements, elemToRef, + conflicts); + } + } + + // Make sure we can delete the send buffers + NeighborRefinementMessage::WaitAllSent(send_ref); + + CheckRefinementMaster(refinements, elemToRef, conflicts); + + const bool conflict = conflicts.size() > 0; + bool globalConflict = false; + MPI_Allreduce(&conflict, &globalConflict, 1, MFEM_MPI_CXX_BOOL, MPI_LOR, + MyComm); + return globalConflict; +} + +int GetHexFaceDir(int face) +{ + // Hexahedron face vertices + // From Geometry::Constants::FaceVert[6][4] in fem/geom.cpp + // {3, 2, 1, 0}, {0, 1, 5, 4}, {1, 2, 6, 5}, + // {2, 3, 7, 6}, {3, 0, 4, 7}, {4, 5, 6, 7} + constexpr std::array hexFaceDir = {2, 1, 0, 1, 0, 2}; + return hexFaceDir[face]; +} + +char GetHexFaceRefType(const bool (&refDir)[3], int face) +{ + const int faceDir = GetHexFaceDir(face); + std::array faceRefDir; + int cnt = 0; + for (int d=0; d<3; ++d) + { + if (d != faceDir) + { + faceRefDir[cnt] = refDir[d] ? 1 : 0; + cnt++; + } + } + + const char ref_type = (char)(faceRefDir[0] + (2 * faceRefDir[1])); + return ref_type; +} + +// Assuming a vertical split of the master face with ordered vertices +// (vn1, vn2, vn3, vn4), check whether there is a horizontal split among the +// slave faces of this face. This recursive function is similar to +// NCMesh::CheckAnisoFace. +bool ParNCMesh::CheckRefAnisoFaceSplits(int vn1, int vn2, int vn3, int vn4, + int level) +{ + const int mid23 = FindMidEdgeNode(vn2, vn3); + const int mid41 = FindMidEdgeNode(vn4, vn1); + + if (mid23 >= 0 && mid41 >= 0) // If horizontally split + { + const int midf = nodes.FindId(mid23, mid41); + if (midf >= 0) + { + if (CheckRefAnisoFaceSplits(vn1, vn2, mid23, mid41, level + 1)) + { + return true; + } + if (CheckRefAnisoFaceSplits(mid41, mid23, vn3, vn4, level + 1)) + { + return true; + } + } + } + + if (level > 0) { return true; } + + return false; +} + +void ParNCMesh::CheckRefinementMaster(const Array &refinements, + const std::map &elemToRef, + std::set &conflicts) +{ + MFEM_VERIFY(Dim == 3, ""); + const NCList &faceList = GetFaceList(); + + for (const auto &mf : faceList.masters) + { + // Check for conflicts only if the master element is marked for refinement + if (elemToRef.count(mf.element) == 0) { continue; } + + const int refIndex = elemToRef.at(mf.element); + const Refinement& ref = refinements[refIndex]; + + bool refDir[3]; + for (int i=0; i<3; ++i) + refDir[i] = ref.s[i] > real_t{0}; + + const char faceRefType = GetHexFaceRefType(refDir, mf.local); + if (faceRefType == 0) { continue; } // No refinement on this face + + std::array fv; + for (int i=0; i<4; ++i) + { + fv[i] = elements[mf.element].node[ + Geometry::Constants::FaceVert[mf.local][i]]; + } + + if (faceRefType != 2) // X or XY split w.r.t. the face. + { + // Check X face split + if (CheckRefAnisoFaceSplits(fv[0], fv[1], fv[2], fv[3])) + { + conflicts.insert(refIndex); + } + } + + if (faceRefType != 1) // Y or XY split w.r.t. the face. + { + // Check Y face split + if (CheckRefAnisoFaceSplits(fv[1], fv[2], fv[3], fv[0])) + { + conflicts.insert(refIndex); + } + } + } +} + +int FindHexFace(const int* no, int vn1, int vn2, int vn3, int vn4) +{ + std::set v; + v.insert({vn1, vn2, vn3, vn4}); + + int face = -1; + for (int f=0; f<6; ++f) + { + bool allFound = true; + for (int i=0; i<4; ++i) + { + const int vi = no[Geometry::Constants::FaceVert[f][i]]; + if (v.count(vi) == 0) + { + allFound = false; + } + } + + if (allFound) + { + MFEM_ASSERT(face == -1, ""); + face = f; + } + } + + MFEM_ASSERT(face >= 0, ""); + return face; +} + +// Assumption: v1 and v2 are indices of hex vertices connected by an edge. +// The return value is {0,1,2} denoting split {X,Y,Z}. +int GetHexEdgeSplit(const int* nodes, int v1, int v2) +{ + Array v(2); + v[0] = v1; + v[1] = v2; + v.Sort(); + + // Find the edge in the hexahedron + int edge = -1; + Array ev(2); + for (int i=0; i<12; ++i) + { + for (int j=0; j<2; ++j) + { + ev[j] = nodes[Geometry::Constants::Edges[i][j]]; + } + ev.Sort(); + + if (ev == v) + { + MFEM_ASSERT(edge == -1, ""); + edge = i; + } + } + + MFEM_ASSERT(edge >= 0, ""); + + constexpr int edgeDir[12] = {0, 1, 0, 1, 0, 1, 0, 1, 2, 2, 2, 2}; + return edgeDir[edge]; +} + +void ParNCMesh::CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4, + const Array &refinements, + const std::map &elemToRef, + std::set &conflicts) +{ + Face* face = faces.Find(vn1, vn2, vn3, vn4); + if (!face) { return; } + + // Find the neighbor of this face. + const int nghbIndex = face->elem[0] == elem ? face->elem[1] : face->elem[0]; + if (nghbIndex < 0) { return; } + + Element &nghb = elements[nghbIndex]; + MFEM_ASSERT(nghb.ref_type == 0, ""); + + if (elemToRef.count(nghbIndex) > 0) + { + const int refIndex = elemToRef.at(nghbIndex); + const Refinement& ref = refinements[refIndex]; + + bool refDir[3]; + for (int i=0; i<3; ++i) + refDir[i] = ref.s[i] > real_t{0}; + + const int localFace = FindHexFace(nghb.node, vn1, vn2, vn3, vn4); + const int faceDir = GetHexFaceDir(localFace); + const char face_ref_type = GetHexFaceRefType(refDir, localFace); + const bool faceAniso = face_ref_type == 1 || + face_ref_type == 2; // X or Y w.r.t. the face. + + if (faceAniso) + { + // Determine whether the face is anisotropically split in the vertical + // direction, with respect to the vertex ordering (vn1, vn2, vn3, vn4). + int hexSplitOnFace = -1; + + const int firstFaceDir = face_ref_type == 1 ? 0 : 1; + + int cnt = 0; + for (int i=0; i<3; ++i) + { + if (i == faceDir) { continue; } + + if (firstFaceDir == cnt) + { + MFEM_ASSERT(hexSplitOnFace == -1, ""); + hexSplitOnFace = i; + } + + cnt++; + } + MFEM_ASSERT(cnt == 2 && hexSplitOnFace >= 0, ""); + + const int edgeSplit = GetHexEdgeSplit(nghb.node, vn1, vn2); + if (edgeSplit != hexSplitOnFace) { conflicts.insert(refIndex); } + } + } + // The else case is that the neighbor is not refined, so there is no need to + // check for conflicts. +} + +void ParNCMesh::CheckRefIsoFace(int elem, int vn1, int vn2, int vn3, int vn4, + int en1, int en2, int en3, int en4, + const Array &refinements, + const std::map &elemToRef, + std::set &conflicts) +{ + CheckRefAnisoFace(elem, vn1, vn2, en2, en4, refinements, elemToRef, conflicts); + CheckRefAnisoFace(elem, en4, en2, vn3, vn4, refinements, elemToRef, conflicts); + CheckRefAnisoFace(elem, vn4, vn1, en1, en3, refinements, elemToRef, conflicts); + CheckRefAnisoFace(elem, en3, en1, vn2, vn3, refinements, elemToRef, conflicts); +} + +void ParNCMesh::CheckRefinement(int elem, char ref_type, + const Array &refinements, + const std::map &elemToRef, + std::set &conflicts) +{ + const Element &el = elements[elem]; + MFEM_ASSERT(el.geom == Geometry::CUBE && el.ref_type == 0, + "Element must be an unrefined hexahedron"); + + const int* no = el.node; + + // Check the faces of this element being refined (depends on ref_type). + // This follows the logic of NCMesh::RefineElement(). + if (ref_type == Refinement::X) // split along X axis + { + CheckRefAnisoFace(elem, no[0], no[1], no[5], no[4], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[2], no[3], no[7], no[6], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[4], no[5], no[6], no[7], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[3], no[2], no[1], no[0], refinements, + elemToRef, conflicts); + } + else if (ref_type == Refinement::Y) // split along Y axis + { + CheckRefAnisoFace(elem, no[1], no[2], no[6], no[5], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[3], no[0], no[4], no[7], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[5], no[6], no[7], no[4], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[0], no[3], no[2], no[1], refinements, + elemToRef, conflicts); + } + else if (ref_type == Refinement::Z) // split along Z axis + { + CheckRefAnisoFace(elem, no[4], no[0], no[1], no[5], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[5], no[1], no[2], no[6], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[6], no[2], no[3], no[7], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[7], no[3], no[0], no[4], refinements, + elemToRef, conflicts); + } + else if (ref_type == Refinement::XY) // XY split + { + CheckRefAnisoFace(elem, no[0], no[1], no[5], no[4], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[1], no[2], no[6], no[5], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[2], no[3], no[7], no[6], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[3], no[0], no[4], no[7], refinements, + elemToRef, conflicts); + + const int mid01 = GetMidEdgeNode(no[0], no[1]); + const int mid12 = GetMidEdgeNode(no[1], no[2]); + const int mid23 = GetMidEdgeNode(no[2], no[3]); + const int mid30 = GetMidEdgeNode(no[3], no[0]); + + const int mid45 = GetMidEdgeNode(no[4], no[5]); + const int mid56 = GetMidEdgeNode(no[5], no[6]); + const int mid67 = GetMidEdgeNode(no[6], no[7]); + const int mid74 = GetMidEdgeNode(no[7], no[4]); + + CheckRefIsoFace(elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01, + mid30, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67, + mid74, refinements, elemToRef, conflicts); + } + else if (ref_type == Refinement::XZ) // XZ split + { + CheckRefAnisoFace(elem, no[3], no[2], no[1], no[0], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[2], no[6], no[5], no[1], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[6], no[7], no[4], no[5], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[7], no[3], no[0], no[4], refinements, + elemToRef, conflicts); + + const int mid01 = GetMidEdgeNode(no[0], no[1]); + const int mid23 = GetMidEdgeNode(no[2], no[3]); + const int mid45 = GetMidEdgeNode(no[4], no[5]); + const int mid67 = GetMidEdgeNode(no[6], no[7]); + + const int mid04 = GetMidEdgeNode(no[0], no[4]); + const int mid15 = GetMidEdgeNode(no[1], no[5]); + const int mid26 = GetMidEdgeNode(no[2], no[6]); + const int mid37 = GetMidEdgeNode(no[3], no[7]); + + CheckRefIsoFace(elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45, + mid04, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67, + mid26, refinements, elemToRef, conflicts); + } + else if (ref_type == Refinement::YZ) // YZ split + { + const int mid12 = GetMidEdgeNode(no[1], no[2]); + const int mid30 = GetMidEdgeNode(no[3], no[0]); + const int mid56 = GetMidEdgeNode(no[5], no[6]); + const int mid74 = GetMidEdgeNode(no[7], no[4]); + + const int mid04 = GetMidEdgeNode(no[0], no[4]); + const int mid15 = GetMidEdgeNode(no[1], no[5]); + const int mid26 = GetMidEdgeNode(no[2], no[6]); + const int mid37 = GetMidEdgeNode(no[3], no[7]); + + CheckRefAnisoFace(elem, no[4], no[0], no[1], no[5], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[0], no[3], no[2], no[1], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[3], no[7], no[6], no[2], refinements, + elemToRef, conflicts); + CheckRefAnisoFace(elem, no[7], no[4], no[5], no[6], refinements, + elemToRef, conflicts); + + CheckRefIsoFace(elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56, + mid15, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74, + mid37, refinements, elemToRef, conflicts); + } + else if (ref_type == Refinement::XYZ) // XYZ split + { + const int mid01 = GetMidEdgeNode(no[0], no[1]); + const int mid12 = GetMidEdgeNode(no[1], no[2]); + const int mid23 = GetMidEdgeNode(no[2], no[3]); + const int mid30 = GetMidEdgeNode(no[3], no[0]); + + const int mid45 = GetMidEdgeNode(no[4], no[5]); + const int mid56 = GetMidEdgeNode(no[5], no[6]); + const int mid67 = GetMidEdgeNode(no[6], no[7]); + const int mid74 = GetMidEdgeNode(no[7], no[4]); + + const int mid04 = GetMidEdgeNode(no[0], no[4]); + const int mid15 = GetMidEdgeNode(no[1], no[5]); + const int mid26 = GetMidEdgeNode(no[2], no[6]); + const int mid37 = GetMidEdgeNode(no[3], no[7]); + + CheckRefIsoFace(elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01, + mid30, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45, + mid04, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56, + mid15, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67, + mid26, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74, + mid37, refinements, elemToRef, conflicts); + CheckRefIsoFace(elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67, + mid74, refinements, elemToRef, conflicts); + } + else + { + MFEM_ABORT("Invalid refinement type."); + } +} void ParNCMesh::Refine(const Array &refinements) { @@ -1513,14 +2014,14 @@ void ParNCMesh::Refine(const Array &refinements) return; } - for (int i = 0; i < refinements.Size(); i++) + for (int i = 0; i < refinements.Size() && Iso; i++) { const Refinement &ref = refinements[i]; - MFEM_VERIFY(ref.GetType() == 7 || Dim < 3, - "anisotropic parallel refinement not supported yet in 3D."); + if (ref.GetType() != Refinement::XYZ) + { + Iso = false; + } } - MFEM_VERIFY(Iso || Dim < 3, - "parallel refinement of 3D aniso meshes not supported yet."); NeighborRefinementMessage::Map send_ref; @@ -1541,7 +2042,7 @@ void ParNCMesh::Refine(const Array &refinements) { const Refinement &ref = refinements[i]; MFEM_ASSERT(ref.index < NElements, ""); - int elem = leaf_elements[ref.index]; + const int elem = leaf_elements[ref.index]; ElementNeighborProcessors(elem, ranks); for (int j = 0; j < ranks.Size(); j++) { diff --git a/mesh/pncmesh.hpp b/mesh/pncmesh.hpp index 5d5c232dc6..d42778086f 100644 --- a/mesh/pncmesh.hpp +++ b/mesh/pncmesh.hpp @@ -86,6 +86,12 @@ public: date. */ void Refine(const Array &refinements) override; + /** See Mesh::AnisotropicConflict. The return value is globally MPI-reduced, + whereas @a conflicts contains local indices of conflicting entries of + @a refinements. */ + bool AnisotropicConflict(const Array &refinements, + std::set &conflicts); + /// Parallel version of NCMesh::LimitNCLevel. void LimitNCLevel(int max_nc_level) override; @@ -211,8 +217,12 @@ public: // utility + /// Return the MPI rank for this process. int GetMyRank() const { return MyRank; } + /// Return true if using more than one MPI process. + bool IsParallel() const override { return NRanks > 1; } + /// Use the communication pattern from last Rebalance() to send element DOFs. void SendRebalanceDofs(int old_ndofs, const Table &old_element_dofs, long old_global_offset, FiniteElementSpace* space); @@ -586,6 +596,44 @@ protected: // implementation std::size_t GroupsMemoryUsage() const; + // The following functions help with checking for anisotropic refinements in + // different directions on a face shared by two hexahedral elements. + + /** For the face with ordered vertices vn* and neighboring element @a elem, + check whether the other neighboring element (if it exists) is marked for + a horizontal refinement conflicting with a vertical split. */ + void CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4, + const Array &refinements, + const std::map &elemToRef, + std::set &conflicts); + + /** For the face with ordered vertices vn*, edge midpoints en*, and + neighboring element @a elem, check whether the other neighboring element + (if it exists) is marked for a refinement conflicting with an isotropic + refinement of the face. */ + void CheckRefIsoFace(int elem, int vn1, int vn2, int vn3, int vn4, + int en1, int en2, int en3, int en4, + const Array &refinements, + const std::map &elemToRef, + std::set &conflicts); + + /// Check whether any master face is marked for a conflicting refinement. + void CheckRefinementMaster(const Array &refinements, + const std::map &elemToRef, + std::set &conflicts); + + /** Check whether the refinement of the element with index @a elem and type + @a ref_type would cause a conflict. */ + void CheckRefinement(int elem, char ref_type, + const Array &refinements, + const std::map &elemToRef, + std::set &conflicts); + + /** For a vertical split of the master face with ordered vertices + (vn1, vn2, vn3, vn4), check whether there is a horizontal split among the + slave faces. */ + bool CheckRefAnisoFaceSplits(int vn1, int vn2, int vn3, int vn4, + int level = 0); friend class NeighborRowMessage; friend class NeighborOrderMessage; }; diff --git a/mesh/spacing.cpp b/mesh/spacing.cpp index 5b3733ff59..c417ca1f87 100644 --- a/mesh/spacing.cpp +++ b/mesh/spacing.cpp @@ -66,6 +66,13 @@ std::unique_ptr GetSpacingFunction(const SpacingType new PiecewiseSpacingFunction(ipar[0], ipar[1], (bool) ipar[2], relN, iparsub, dpar)); + case SpacingType::PARTIAL: + MFEM_VERIFY(ipar.Size() >= 8, "Invalid spacing function parameters"); + ipar.GetSubArray(8, ipar.Size() - 8, iparsub); + return std::unique_ptr( + new PartialSpacingFunction(ipar[0], ipar[1], ipar[2], + ipar[3], ipar[4], iparsub, + dpar, (SpacingType) ipar[5])); default: MFEM_ABORT("Unknown spacing type \"" << int(spacingType) << "\""); break; @@ -90,7 +97,7 @@ void GeometricSpacingFunction::CalculateSpacing() // Find the root of g(r) = s * (r^n - 1) - r + 1 by Newton's method. constexpr real_t convTol = 1.0e-8; - constexpr int maxIter = 20; + constexpr int maxIter = 100; const real_t s_unif = 1.0 / ((real_t) n); @@ -471,7 +478,7 @@ void PiecewiseSpacingFunction::SetupPieces(Array const& ipar, void PiecewiseSpacingFunction::ScaleParameters(real_t a) { - for (auto& p : pieces) { p->ScaleParameters(a); } + for (auto &p : pieces) { p->ScaleParameters(a); } } void PiecewiseSpacingFunction::Print(std::ostream &os) const @@ -498,6 +505,8 @@ void PiecewiseSpacingFunction::Print(std::ostream &os) const Array ipar; for (auto& p : pieces) { + MFEM_VERIFY(p->GetSpacingType() != SpacingType::PIECEWISE, + "Piecewise spacings should not be composed"); os << "\n" << int(p->GetSpacingType()) << " " << p->NumIntParameters() << " " << p->NumDoubleParameters(); @@ -597,7 +606,7 @@ void PiecewiseSpacingFunction::CalculateSpacing() bool PiecewiseSpacingFunction::Nested() const { - for (auto& p : pieces) + for (const auto &p : pieces) { if (!p->Nested()) { @@ -608,4 +617,82 @@ bool PiecewiseSpacingFunction::Nested() const return true; } +void PartialSpacingFunction::SetupFull(SpacingType typeFull, + Array const& ipar, + Vector const& dpar) +{ + fullSpacing = GetSpacingFunction(typeFull, ipar, dpar); } + +void PartialSpacingFunction::CalculateSpacing() +{ + s.SetSize(n); + + if (n == 1) + { + s[0] = 1.0; + fullSpacing->SetSize(1); + return; + } + + const int ref = n / num_elems; + MFEM_VERIFY(ref * num_elems == n, "Invalid number of elements"); + + fullSpacing->SetSize(ref * num_elems_full); + + const int os = ref * first_elem; + for (int i = 0; i < n; ++i) + { + s[i] = fullSpacing->Eval(os + i); + } + + // Normalize + const double d1 = s.Sum(); + for (int i = 0; i < n; ++i) + { + s[i] /= d1; + } +} + +void PartialSpacingFunction::ScaleParameters(real_t a) +{ + fullSpacing->ScaleParameters(a); +} + +void PartialSpacingFunction::Print(std::ostream &os) const +{ + os << int(SpacingType::PARTIAL) << " " << NumIntParameters() << " " + << NumDoubleParameters() << " " << n << " " << (int) reverse << "\n" + << first_elem << " " << num_elems << " " << num_elems_full << "\n"; + + // Write integer parameters for the full spacing. + Array ipar; + os << int(fullSpacing->GetSpacingType()) << " " + << fullSpacing->NumIntParameters() << " " + << fullSpacing->NumDoubleParameters(); + + fullSpacing->GetIntParameters(ipar); + + for (auto& ip : ipar) + { + os << " " << ip; + } + + os << "\n"; + + // Write double parameters for the full spacing. + Vector dpar; + fullSpacing->GetDoubleParameters(dpar); + + if (dpar.Size() > 0) + { + for (auto dp : dpar) + { + os << dp << " "; + } + } + + os << "\n"; +} + +} // namespace mfem diff --git a/mesh/spacing.hpp b/mesh/spacing.hpp index 3f4b8282ad..0ab30c4a17 100644 --- a/mesh/spacing.hpp +++ b/mesh/spacing.hpp @@ -21,7 +21,7 @@ namespace mfem { enum class SpacingType {UNIFORM_SPACING, LINEAR, GEOMETRIC, BELL, - GAUSSIAN, LOGARITHMIC, PIECEWISE + GAUSSIAN, LOGARITHMIC, PIECEWISE, PARTIAL }; /// Class for spacing functions that define meshes in a dimension, using a @@ -31,9 +31,9 @@ class SpacingFunction public: /** @brief Base class constructor. @param[in] n Size or number of intervals, which defines elements. - @param[in] r Whether to reverse the spacings, false by default. - @param[in] s Whether to scale parameters by the refinement or coarsening - factor, in the function @a SpacingFunction::ScaleParameters. + @param[in] r Whether to reverse the spacings, false by default. + @param[in] s Whether to scale parameters by the refinement or coarsening + factor, in the function @a SpacingFunction::ScaleParameters. */ SpacingFunction(int n, bool r=false, bool s=false) : n(n), reverse(r), scale(s) { } @@ -47,6 +47,10 @@ public: /// Sets the property that determines whether the spacing is reversed. void SetReverse(bool r) { reverse = r; } + bool GetReverse() const { return reverse; } + + void Flip() { reverse = !reverse; } + /// Returns the width of interval @a p (between 0 and @a Size() - 1). virtual real_t Eval(int p) const = 0; @@ -97,6 +101,13 @@ public: /// Returns a clone (deep-copy) of this spacing function. virtual std::unique_ptr Clone() const; + void FullyCoarsen() + { + const int nfine = n; + SetSize(1); + ScaleParameters(nfine); + } + virtual ~SpacingFunction() = default; protected: @@ -447,7 +458,7 @@ private: class GaussianSpacingFunction : public SpacingFunction { public: - /** @brief Constructor for BellSpacingFunction. + /** @brief Constructor for GaussianSpacingFunction. @param[in] n Size or number of intervals, which defines elements. @param[in] r Whether to reverse the spacings. @param[in] s0 Width of the first interval (element). @@ -639,7 +650,7 @@ public: /// Copy constructor (deep-copy all data, including SpacingFunction pieces) PiecewiseSpacingFunction(const PiecewiseSpacingFunction &sf) - : SpacingFunction(sf), np(sf.np), partition(sf.partition), + : SpacingFunction(sf.n, sf.reverse), np(sf.np), partition(sf.partition), npartition(sf.npartition), pieces(), n0(sf.n0), s(sf.s) { // To copy, the pointers must be cloned. @@ -681,22 +692,79 @@ public: void SetupPieces(Array const& ipar, Vector const& dpar); SpacingType GetSpacingType() const override { return SpacingType::PIECEWISE; } - int NumIntParameters() const override { return 3; } - int NumDoubleParameters() const override { return np - 1; } + int NumIntParameters() const override + { + int count = 3 + np; + for (const auto &p : pieces) + { + // Add three for the type and the integer and double parameter counts. + count += p->NumIntParameters() + 3; + } + return count; + } + + int NumDoubleParameters() const override + { + int count = np - 1; + for (const auto &p : pieces) + { + count += p->NumDoubleParameters(); + } + return count; + } + + Array RelativePieceSizes() const { return npartition; } + + void ScalePartition(Array const& f, bool reorient) + { + MFEM_VERIFY(npartition.Size() == f.Size(), ""); + n0 = 0; + for (int i=0; i & p) const override { - p.SetSize(3 + np); + p.SetSize(NumIntParameters()); p[0] = n; p[1] = np; p[2] = (int) reverse; for (int i=0; i ipar; + int os = 3 + np; + for (int i=0; iGetSpacingType(); + p[os++] = pieces[i]->NumIntParameters(); + p[os++] = pieces[i]->NumDoubleParameters(); + + pieces[i]->GetIntParameters(ipar); + for (auto ip : ipar) + { + p[os++] = ip; + } + } } void GetDoubleParameters(Vector & p) const override { - p.SetSize(np - 1); - p = partition; + p.SetSize(NumDoubleParameters()); + for (int i=0; iGetDoubleParameters(dpar); + for (auto dp : dpar) + { + p[os++] = dp; + } + } } // PiecewiseSpacingFunction is nested if and only if all pieces are nested. @@ -705,10 +773,111 @@ public: private: int np; ///< Number of pieces Vector partition; ///< Partition of the unit interval - Array npartition; ///< Number of intervals in each partition + Array npartition; ///< Relative number of intervals in each partition std::vector> pieces; - int n0 = 0; ///< Total number of intervals + int n0 = 0; ///< Sum of npartition + + Vector s; ///< Stores the spacings calculated by @a CalculateSpacing + + /// Calculate parameters used by @a Eval and @a EvalAll + void CalculateSpacing(); +}; + +/** @brief Partial spacing function, defined as part of an existing spacing + function. + */ +class PartialSpacingFunction : public SpacingFunction +{ +public: + PartialSpacingFunction(int n, bool r, int rel_first_elem, int rel_num_elems, + int rel_num_elems_full, + Array const& ipar, Vector const& dpar, + SpacingType typeFull) + : SpacingFunction(n, r), num_elems_full(rel_num_elems_full), + num_elems(rel_num_elems), first_elem(rel_first_elem) + { + SetupFull(typeFull, ipar, dpar); + CalculateSpacing(); + } + + /// Copy constructor (deep-copy all data, including SpacingFunction pieces) + PartialSpacingFunction(const PartialSpacingFunction &sf) + : SpacingFunction(sf.n, sf.reverse), fullSpacing(sf.fullSpacing->Clone()), + num_elems_full(sf.num_elems_full), num_elems(sf.num_elems), + first_elem(sf.first_elem), s(sf.s) + { } + + PartialSpacingFunction& operator=(const PartialSpacingFunction &sf) + { + PartialSpacingFunction tmp(sf); + std::swap(tmp, *this); + return *this; + } + + PartialSpacingFunction(PartialSpacingFunction &&sf) = default; + PartialSpacingFunction& operator=(PartialSpacingFunction &&sf) = default; + + void SetSize(int size) override + { + n = size; + CalculateSpacing(); + } + + real_t Eval(int p) const override + { + const int i = reverse ? n - 1 - p : p; + return s[i]; + } + + void ScaleParameters(real_t a) override; + + void Print(std::ostream &os) const override; + + std::unique_ptr Clone() const override + { + return std::unique_ptr( + new PartialSpacingFunction(*this)); + } + + void SetupFull(SpacingType typeFull, + Array const& ipar, Vector const& dpar); + + SpacingType GetSpacingType() const override { return SpacingType::PARTIAL; } + int NumIntParameters() const override { return 8 + fullSpacing->NumIntParameters(); } + int NumDoubleParameters() const override { return fullSpacing->NumDoubleParameters(); } + + void GetIntParameters(Array & p) const override + { + Array ipar; + fullSpacing->GetIntParameters(ipar); + p.SetSize(8 + ipar.Size()); + p[0] = n; + p[1] = (int) reverse; + p[2] = first_elem; + p[3] = num_elems; + p[4] = num_elems_full; + p[5] = (int) fullSpacing->GetSpacingType(); + p[6] = ipar.Size(); + p[7] = NumDoubleParameters(); + for (int i=0; iGetDoubleParameters(p); + } + + // PartialSpacingFunction is nested if and only if the full spacing is nested. + bool Nested() const override { return fullSpacing->Nested(); } + +private: + std::unique_ptr fullSpacing; + + // The following numbers and indices may be relative, not absolute. + int num_elems_full; ///< Reference number of elements in fullSpacing + int num_elems; ///< Number of elements, relative to num_elems_full + int first_elem; ///< Index of the first element, relative to num_elems_full Vector s; ///< Stores the spacings calculated by @a CalculateSpacing diff --git a/miniapps/autodiff/makefile b/miniapps/autodiff/makefile index 221491b2a9..7bab3a6701 100644 --- a/miniapps/autodiff/makefile +++ b/miniapps/autodiff/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/autodiff/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) diff --git a/miniapps/common/makefile b/miniapps/common/makefile index 3e812c07ef..89b3897917 100644 --- a/miniapps/common/makefile +++ b/miniapps/common/makefile @@ -20,6 +20,7 @@ CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ # Default target all: lib-common +# Include defaults.mk to get the definition of BUILD_SOFLAGS DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk include $(DEFAULTS_MK) @@ -30,13 +31,7 @@ MFEM_LIB_FILE = mfem_is_not_built ifneq (clean,$(MAKECMDGOALS)) -include $(CONFIG_MK) - ifeq ($(MFEM_USE_CUDA),YES) - XLINKER = $(CUDA_XLINKER) - else ifeq ($(MFEM_USE_HIP),YES) - XLINKER = $(HIP_XLINKER) - else - XLINKER = $(CXX_XLINKER) - endif + XLINKER = $(MFEM_XLINKER) BUILD_REAL_DIR = $(realpath .) BUILD_SOFLAGS := $(subst libmfem.,libmfem-common.,$(BUILD_SOFLAGS)) diff --git a/miniapps/diag-smoothers/makefile b/miniapps/diag-smoothers/makefile index c44d7475ff..16d4b3ca02 100644 --- a/miniapps/diag-smoothers/makefile +++ b/miniapps/diag-smoothers/makefile @@ -15,11 +15,9 @@ MFEM_BUILD_DIR ?= ../.. SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/diag-smoothers/,) CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk MFEM_LIB_FILE = mfem_is_not_built --include $(DEFAULTS_MK) -include $(CONFIG_MK) DS_COMMON_SRC = ds-common.cpp @@ -31,8 +29,7 @@ MINIAPPS = $(if $(MFEM_USE_MPI:NO=),$(PAR_MINIAPPS),) COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\ - $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\ - $(MFEM_BUILD_DIR)/miniapps/common)) + $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common)) APP_DEPS = $(DS_COMMON_OBJ) $(MFEM_LIB_FILE) $(CONFIG_MK) APP_LIBS = $(COMMON_LIB) $(MFEM_LIBS) diff --git a/miniapps/dpg/makefile b/miniapps/dpg/makefile index 0c18d29a44..450f34cc9e 100644 --- a/miniapps/dpg/makefile +++ b/miniapps/dpg/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/dpg/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) @@ -69,8 +65,7 @@ COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common # If MFEM_SHARED is set, add the ../common rpath COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\ - $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\ - $(MFEM_BUILD_DIR)/miniapps/common)) + $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common)) .SUFFIXES: .SUFFIXES: .o .cpp .mk diff --git a/miniapps/electromagnetics/CMakeLists.txt b/miniapps/electromagnetics/CMakeLists.txt index 8a80084e6a..d750bc6c78 100644 --- a/miniapps/electromagnetics/CMakeLists.txt +++ b/miniapps/electromagnetics/CMakeLists.txt @@ -34,6 +34,11 @@ if (MFEM_USE_MPI) EXTRA_HEADERS maxwell_solver.hpp ${MFEM_MINIAPPS_COMMON_HEADERS} LIBRARIES mfem-common) + add_mfem_miniapp(lorentz + MAIN lorentz.cpp + EXTRA_HEADERS ${MFEM_MINIAPPS_COMMON_HEADERS} + LIBRARIES mfem-common) + # Add the corresponding tests to the "test" target if (MFEM_ENABLE_TESTING) add_test(NAME tesla_np=4 diff --git a/miniapps/electromagnetics/lorentz.cpp b/miniapps/electromagnetics/lorentz.cpp new file mode 100644 index 0000000000..c3c0f30ad1 --- /dev/null +++ b/miniapps/electromagnetics/lorentz.cpp @@ -0,0 +1,571 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. +// +// ----------------------------------------------------- +// Lorentz Miniapp: Simple Lorentz Force Particle Mover +// ----------------------------------------------------- +// +// This miniapp computes the trajectory of a single charged particle subject to +// Lorentz forces. +// +// dp/dt = q (E + v x B) +// +// The method used is the explicit Boris algortihm which conserves phase space +// volume for long term accuracy. +// +// The electric and magnetic fields are read from VisItDataCollection objects +// such as those produced by the Volta and Tesla miniapps. It is notable that +// these two fields do not need to be defined on the same mesh. Of course, the +// particle trajectory can only be computed on the intersection of the two +// domains. The starting point of the path must be chosen within in this +// intersection and the trajectory will terminate when it leaves the +// intersection or reaches a specified time duration. +// +// Note that the VisItDataCollection objects must have been stored using the +// parallel format e.g. visit_dc.SetFormat(DataCollection::PARALLEL_FORMAT);. +// Without this optional format specifier the vector field lookups will fail. +// +// Compile with: make lorentz +// +// Sample runs: +// +// Free particle moving with constant velocity +// mpirun -np 4 lorentz -p0 '1 1 1' +// +// Particle accelerating in a constant electric field +// mpirun -np 4 volta -m ../../data/inline-hex.mesh -dbcs '1 6' -dbcv '0 1' +// mpirun -np 4 lorentz -er Volta-AMR-Parallel -x0 '0.5 0.5 0.9' -p0 '1 0 0' +// +// Particle accelerating in a constant magnetic field +// mpirun -np 4 tesla -m ../../data/inline-hex.mesh -ubbc '0 0 1' +// mpirun -np 4 lorentz -br Tesla-AMR-Parallel -x0 '0.1 0.5 0.1' -p0 '0 0.4 0.1' -tf 9 +// +// Magnetic mirror effect near a charged sphere and a bar magnet +// mpirun -np 4 volta -m ../../data/ball-nurbs.mesh -dbcs 1 -cs '0 0 0 0.1 2e-11' -rs 2 -maxit 4 +// mpirun -np 4 tesla -m ../../data/fichera.mesh -maxit 4 -rs 3 -bm '-0.1 -0.1 -0.1 0.1 0.1 0.1 0.1 -1e10' +// mpirun -np 4 lorentz -er Volta-AMR-Parallel -ec 4 -br Tesla-AMR-Parallel -bc 4 -x0 '0.8 0 0' -p0 '-8 -4 4' -q -10 -tf 0.2 -dt 1e-3 -rf 1e-6 +// +// This miniapp demonstrates the use of the ParMesh::FindPoints functionality +// to evaluate field data from stored DataCollection objects. While this +// miniapp is far from a full particle-in-cell (PIC) code it does demonstrate +// some of the building blocks that might be used to construct the particle +// mover portion of a PIC code. + +#include "mfem.hpp" +#include "../common/fem_extras.hpp" +#include "../common/pfem_extras.hpp" +#include "electromagnetics.hpp" +#include +#include + +using namespace std; +using namespace mfem; +using namespace mfem::common; +using namespace mfem::electromagnetics; + +typedef DataCollection::FieldMapType fields_t; + +/// This class implements the Boris algorithm as described in the +/// article `Why is Boris algorithm so good?` by H. Qin et al in +/// Physics of Plasmas, Volume 20 Issue 8, August 2013, +/// https://doi.org/10.1063/1.4818428. +class BorisAlgorithm +{ +private: + real_t charge_; + real_t mass_; + + ParMesh *E_pmesh_; + ParGridFunction *E_field_; + + ParMesh *B_pmesh_; + ParGridFunction *B_field_; + + mutable Array elem_id_; + mutable Array ip_; + + mutable Vector E_; + mutable Vector B_; + mutable Vector pxB_; + mutable Vector pm_; + mutable Vector pp_; + + // Returns true if a usable V has been found. If @a pgf is NULL, V = 0 is + // returned as a default value. + bool GetValue(ParMesh *pmesh, ParGridFunction *pgf, Vector q, Vector &V) + { + DenseMatrix point(q.GetData(), 3, 1); + + int pt_found = + (pmesh != NULL) ? pmesh->FindPoints(point, elem_id_, ip_, false) : -1; + + // We have a mesh but the point was not found. The path must be outside + // the domain of interest. + if (pmesh != NULL && pt_found <= 0) { return false; } + + int pt_root = -1; + + if (pt_found > 0 && elem_id_[0] >= 0 && pgf != NULL) + { + pt_root = pmesh->GetMyRank(); + + pgf->GetVectorValue(elem_id_[0], ip_[0], V); + } + else + { + pt_root = 0; + V = 0.0; + } + + // Determine processor which found the field point + int glb_pt_root = -1; + MPI_Allreduce(&pt_root, &glb_pt_root, 1, + MPI_INT, MPI_MAX, MPI_COMM_WORLD); + + // Send the field value to the root processor + if (pmesh != NULL && elem_id_[0] >= 0 && glb_pt_root != 0) + { + MPI_Send(V.GetData(), 3, MPITypeMap::mpi_type, + 0, 1030, MPI_COMM_WORLD); + } + + // Receive the field value on the root processor + if (Mpi::Root() && pmesh != NULL && glb_pt_root != 0) + { + MPI_Status status; + MPI_Recv(V.GetData(), 3, MPITypeMap::mpi_type, + glb_pt_root, 1030, MPI_COMM_WORLD, &status); + } + return true; + } + +public: + BorisAlgorithm(ParGridFunction *E_gf, + ParGridFunction *B_gf, + real_t charge, real_t mass) + : charge_(charge), mass_(mass), + E_field_(E_gf), + B_field_(B_gf), + E_(3), B_(3), pxB_(3), pm_(3), pp_(3) + { + E_pmesh_ = (E_field_) ? E_field_->ParFESpace()->GetParMesh() : NULL; + B_pmesh_ = (B_field_) ? B_field_->ParFESpace()->GetParMesh() : NULL; + } + + bool Step(Vector &q, Vector &p, real_t &t, real_t &dt) + { + // Locate current point in each mesh, evaluate the fields, and collect + // field values on the root processor. + if (!GetValue(E_pmesh_, E_field_, q, E_)) { return false; } + if (!GetValue(B_pmesh_, B_field_, q, B_)) { return false; } + + // Compute updated position and momentum using the Boris algorithm + if (Mpi::Root()) + { + // Compute half of the contribution from q E + add(p, 0.5 * dt * charge_, E_, pm_); + + // Compute the contributiobn from q p x B + const real_t B2 = B_ * B_; + + // ... along pm x B + const real_t a1 = 4.0 * dt * charge_ * mass_; + pm_.cross3D(B_, pxB_); + pp_.Set(a1, pxB_); + + // ... along pm + const real_t a2 = 4.0 * mass_ * mass_ - + dt * dt * charge_ * charge_ * B2; + pp_.Add(a2, pm_); + + // ... along B + const real_t a3 = 2.0 * dt * dt * charge_ * charge_ * (B_ * pm_); + pp_.Add(a3, B_); + + // scale by common denominator + const real_t a4 = 4.0 * mass_ * mass_ + + dt * dt * charge_ * charge_ * B2; + pp_ /= a4; + + // Update the momentum + add(pp_, 0.5 * dt * charge_, E_, p); + + // Update the position + q.Add(dt / mass_, p); + } + + // Update the time + t += dt; + + // Broadcast the updated position + MPI_Bcast(q.GetData(), 3, MPITypeMap::mpi_type, + 0, MPI_COMM_WORLD); + + // Broadcast the updated momentum + MPI_Bcast(p.GetData(), 3, MPITypeMap::mpi_type, + 0, MPI_COMM_WORLD); + + return true; + } +}; + +// Open the named VisItDataCollection and read the named field. +// Returns pointers to the two new objects. +int ReadGridFunction(const char * coll_name, const char * field_name, + int pad_digits_cycle, int pad_digits_rank, int cycle, + VisItDataCollection *&dc, ParGridFunction *& gf); + +// By default the initial position will be the center of the intersection +// of the bounding boxes of the meshes containing the E and B fields. +void SetInitialPosition(VisItDataCollection *E_dc, + VisItDataCollection *B_dc, + Vector &x_init); + +// Build a quadrilateral mesh approximating the trajectory as a +// ribbon. One edge of the ribbon follows the trajectory of the +// particle. The opposite edge is offset by the acceleration vector +// (scaled by a constant called the r_factor). +Mesh MakeTrajectoryMesh(int step, real_t m, real_t dt, real_t r_factor, + const DenseMatrix &pos_data, + const DenseMatrix &mom_data); + +// Prints the program's logo to the given output stream +void display_banner(ostream & os); + +int main(int argc, char *argv[]) +{ + Mpi::Init(argc, argv); + Hypre::Init(); + + if ( Mpi::Root() ) { display_banner(cout); } + + const char *E_coll_name = ""; + const char *E_field_name = "E"; + int E_cycle = 10; + int E_pad_digits_cycle = 6; + int E_pad_digits_rank = 6; + + const char *B_coll_name = ""; + const char *B_field_name = "B"; + int B_cycle = 10; + int B_pad_digits_cycle = 6; + int B_pad_digits_rank = 6; + + real_t q = 1.0; + real_t m = 1.0; + real_t dt = 1e-2; + real_t t_init = 0.0; + real_t t_final = 1.0; + real_t r_factor = -1.0; + Vector x_init; + Vector p_init; + int visport = 19916; + bool visualization = true; + bool visit = true; + + OptionsParser args(argc, argv); + args.AddOption(&E_coll_name, "-er", "--e-root-file", + "Set the VisIt data collection E field root file prefix."); + args.AddOption(&E_field_name, "-ef", "--e-field-name", + "Set the VisIt data collection E field name"); + args.AddOption(&E_cycle, "-ec", "--e-cycle", + "Set the E field cycle index to read."); + args.AddOption(&E_pad_digits_cycle, "-epdc", "--e-pad-digits-cycle", + "Number of digits in E field cycle."); + args.AddOption(&E_pad_digits_rank, "-epdr", "--e-pad-digits-rank", + "Number of digits in E field MPI rank."); + args.AddOption(&B_coll_name, "-br", "--b-root-file", + "Set the VisIt data collection B field root file prefix."); + args.AddOption(&B_field_name, "-bf", "--b-field-name", + "Set the VisIt data collection B field name"); + args.AddOption(&B_cycle, "-bc", "--b-cycle", + "Set the B field cycle index to read."); + args.AddOption(&B_pad_digits_cycle, "-bpdc", "--b-pad-digits-cycle", + "Number of digits in B field cycle."); + args.AddOption(&B_pad_digits_rank, "-bpdr", "--b-pad-digits-rank", + "Number of digits in B field MPI rank."); + args.AddOption(&q, "-q", "--charge", + "Particle charge."); + args.AddOption(&m, "-m", "--mass", + "Particle mass."); + args.AddOption(&dt, "-dt", "--time-step", + "Time Step."); + args.AddOption(&t_init, "-ti", "--initial-time", + "Initial Time."); + args.AddOption(&t_final, "-tf", "--final-time", + "Final Time."); + args.AddOption(&x_init, "-x0", "--initial-position", + "Initial position."); + args.AddOption(&p_init, "-p0", "--initial-momentum", + "Initial momentum."); + args.AddOption(&r_factor, "-rf", "--ribbon-factor", + "Scale factor for ribbon width (rf * (p1-p0) / (m * dt) " + "where p0 and p1 are computed momenta)."); + args.AddOption(&visualization, "-vis", "--visualization", "-no-vis", + "--no-visualization", + "Enable or disable GLVis visualization."); + args.AddOption(&visit, "-visit", "--visit", "-no-visit", "--no-visit", + "Enable or disable VisIt visualization."); + args.AddOption(&visport, "-p", "--send-port", "Socket for GLVis."); + args.Parse(); + if (!args.Good()) + { + if (Mpi::Root()) + { + args.PrintUsage(cout); + } + return 1; + } + if (r_factor <= 0.0) + { + r_factor = dt; + } + if (Mpi::Root()) + { + args.PrintOptions(cout); + } + + VisItDataCollection *E_dc = NULL; + ParGridFunction *E_gf = NULL; + + if (strcmp(E_coll_name, "")) + { + if (ReadGridFunction(E_coll_name, E_field_name, E_pad_digits_cycle, + E_pad_digits_rank, E_cycle, E_dc, E_gf)) + { + mfem::out << "Error loading E field" << endl; + return 1; + } + } + + VisItDataCollection *B_dc = NULL; + ParGridFunction *B_gf = NULL; + + if (strcmp(B_coll_name, "")) + { + if (ReadGridFunction(B_coll_name, B_field_name, B_pad_digits_cycle, + B_pad_digits_rank, B_cycle, B_dc, B_gf)) + { + mfem::out << "Error loading B field" << endl; + return 1; + } + } + + if (x_init.Size() < 3) + { + SetInitialPosition(E_dc, B_dc, x_init); + } + if (p_init.Size() < 3) + { + p_init.SetSize(3); p_init = 0.0; + } + if (Mpi::Root()) + { + mfem::out << "Initial position: "; x_init.Print(mfem::out); + mfem::out << "Initial momentum: "; p_init.Print(mfem::out); + } + + BorisAlgorithm boris(E_gf, B_gf, q, m); + Vector pos(x_init); + Vector mom(p_init); + + ofstream ofs("Lorentz.dat"); + ofs.precision(14); + + int nsteps = 1 + (int)ceil((t_final - t_init) / dt); + DenseMatrix pos_data(3, nsteps); + DenseMatrix mom_data(3, nsteps + 1); + mom_data.SetCol(0, p_init); + + if (Mpi::Root()) + { + mfem::out << "Maximum number of steps: " << nsteps << endl; + } + + int step = -1; + real_t t = t_init; + do + { + if (Mpi::Root()) + { + ofs << t + << '\t' << pos[0] << '\t' << pos[1] << '\t' << pos[2] + << '\t' << mom[0] << '\t' << mom[1] << '\t' << mom[2] + << '\n'; + } + step++; + + pos_data.SetCol(step, pos); + mom_data.SetCol(step + 1, mom); + } + while (boris.Step(pos, mom, t, dt) && step < nsteps - 1); + + if (Mpi::Root() && (visit || visualization)) + { + Mesh trajectory = MakeTrajectoryMesh(step, m, dt, r_factor, + pos_data, mom_data); + + L2_FECollection fec_l2(0, 2); + FiniteElementSpace fes_l2(&trajectory, &fec_l2); + GridFunction traj_time(&fes_l2); + for (int i=0; i ) | \\/\\ ___/| | \\ | / / " + << endl + << " |_______ \\____/|__| \\___ >___| /__| /_____ \\" + << endl + << " \\/ \\/ \\/ \\/" + << endl << flush; +} + +int ReadGridFunction(const char * coll_name, const char * field_name, + int pad_digits_cycle, int pad_digits_rank, int cycle, + VisItDataCollection *&dc, ParGridFunction *& gf) +{ + dc = new VisItDataCollection(MPI_COMM_WORLD, coll_name); + dc->SetPadDigitsCycle(pad_digits_cycle); + dc->SetPadDigitsRank(pad_digits_rank); + dc->Load(cycle); + + if (dc->Error() != DataCollection::No_Error) + { + mfem::out << "Error loading VisIt data collection: " + << coll_name << endl; + return 1; + } + + if (dc->GetMesh()->Dimension() < 3) + { + mfem::out << "Field must be defined on a three dimensional mesh" + << endl; + return 1; + } + + if (dc->HasField(field_name)) + { + gf = dc->GetParField(field_name); + } + + return 0; +} + +void SetInitialPosition(VisItDataCollection *E_dc, + VisItDataCollection *B_dc, + Vector &x_init) +{ + x_init.SetSize(3); x_init = 0.0; + + if (E_dc != NULL || B_dc != NULL) + { + Vector E_p_min(3); E_p_min = -infinity(); + Vector E_p_max(3); E_p_max = infinity(); + if (E_dc != NULL) + { + ParMesh * E_pmesh = dynamic_cast(E_dc->GetMesh()); + E_pmesh->GetBoundingBox(E_p_min, E_p_max); + } + + Vector B_p_min(3); B_p_min = -infinity(); + Vector B_p_max(3); B_p_max = infinity(); + if (B_dc != NULL) + { + ParMesh *B_pmesh = dynamic_cast(B_dc->GetMesh()); + B_pmesh->GetBoundingBox(B_p_min, B_p_max); + } + + for (int d = 0; d<3; d++) + { + const real_t p_min = std::max(E_p_min[d], B_p_min[d]); + const real_t p_max = std::min(E_p_max[d], B_p_max[d]); + x_init[d] = 0.5 * (p_min + p_max); + } + } +} + +Mesh MakeTrajectoryMesh(int step, real_t m, real_t dt, real_t r_factor, + const DenseMatrix &pos_data, + const DenseMatrix &mom_data) +{ + Mesh trajectory(2, 2 * (step + 1), step, 0, 3); + + for (int i=0; i<=step; i++) + { + trajectory.AddVertex(pos_data(0,i), pos_data(1,i), pos_data(2,i)); + + real_t dpx = (mom_data(0, i + 1) - mom_data(0, i)) / (m * dt); + real_t dpy = (mom_data(1, i + 1) - mom_data(1, i)) / (m * dt); + real_t dpz = (mom_data(2, i + 1) - mom_data(2, i)) / (m * dt); + + trajectory.AddVertex(pos_data(0,i) + r_factor * dpx, + pos_data(1,i) + r_factor * dpy, + pos_data(2,i) + r_factor * dpz); + } + + int v[4]; + for (int i=0; iOwnFEC()) { diff --git a/miniapps/gslib/findpts.cpp b/miniapps/gslib/findpts.cpp index 6684ffef1a..182fdab309 100644 --- a/miniapps/gslib/findpts.cpp +++ b/miniapps/gslib/findpts.cpp @@ -330,8 +330,7 @@ int main (int argc, char *argv[]) // Find and Interpolate FE function values on the desired points. Vector interp_vals(pts_cnt*vec_dim); - FindPointsGSLIB finder; - finder.Setup(mesh); + FindPointsGSLIB finder(mesh); finder.SetL2AvgType(FindPointsGSLIB::NONE); finder.Interpolate(vxyz, field_vals, interp_vals, point_ordering); Array code_out = finder.GetCode(); @@ -374,9 +373,6 @@ int main (int argc, char *argv[]) << "\nPoints not found: " << not_found << "\nPoints on faces: " << face_pts << endl; - // Free the internal gslib data. - finder.FreeData(); - delete fec; return 0; diff --git a/miniapps/gslib/makefile b/miniapps/gslib/makefile index 2fa580e95d..2f1ff4c0ed 100644 --- a/miniapps/gslib/makefile +++ b/miniapps/gslib/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/gslib/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) @@ -42,8 +38,7 @@ COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common # If MFEM_SHARED is set, add the ../common rpath COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\ - $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\ - $(MFEM_BUILD_DIR)/miniapps/common)) + $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common)) .SUFFIXES: .SUFFIXES: .o .cpp .mk diff --git a/miniapps/gslib/pfindpts.cpp b/miniapps/gslib/pfindpts.cpp index 8a8e7b5102..5f6ae41e2d 100644 --- a/miniapps/gslib/pfindpts.cpp +++ b/miniapps/gslib/pfindpts.cpp @@ -455,8 +455,7 @@ int main (int argc, char *argv[]) // Find and Interpolate FE function values on the desired points. Vector interp_vals(pts_cnt*vec_dim); - FindPointsGSLIB finder(MPI_COMM_WORLD); - finder.Setup(pmesh, 1.0, 1e-14); + FindPointsGSLIB finder(pmesh, 1.0, 1e-14); // output the AABB and OBB meshes setup by GSLIB Mesh *aabb_mesh = finder.GetBoundingBoxMesh(0); @@ -507,7 +506,7 @@ int main (int argc, char *argv[]) MPI_Barrier(MPI_COMM_WORLD); // finder.SetDistanceToleranceForPointsFoundOnBoundary(10); - // Enable GPU to CPU fallback for GPUData only if you must use an older + // Enable GPU to CPU fallback for GPUData only if you are using an older // version of GSLIB. // finder.SetGPUtoCPUFallback(true); finder.FindPoints(vxyz, point_ordering); @@ -647,9 +646,6 @@ int main (int argc, char *argv[]) << max_dist << endl; } - // // Free the internal gslib data. - // finder.FreeData(); - delete fec; if (randomization != 0) { delete mesh; } diff --git a/miniapps/gslib/schwarz_ex1.cpp b/miniapps/gslib/schwarz_ex1.cpp index 0f02f6cd7d..28d2a4a173 100644 --- a/miniapps/gslib/schwarz_ex1.cpp +++ b/miniapps/gslib/schwarz_ex1.cpp @@ -325,9 +325,6 @@ int main(int argc, char *argv[]) } } - // Free the used memory. - finder1.FreeData(); - finder2.FreeData(); for (int i = 0; i < nmeshes; i++) { delete a_ar[i]; diff --git a/miniapps/gslib/schwarz_ex1p.cpp b/miniapps/gslib/schwarz_ex1p.cpp index 256c6b4397..343854cd69 100644 --- a/miniapps/gslib/schwarz_ex1p.cpp +++ b/miniapps/gslib/schwarz_ex1p.cpp @@ -308,7 +308,6 @@ int main(int argc, char *argv[]) pmesh->GetNodes()->SetTrueVector(); Vector vxyz = pmesh->GetNodes()->GetTrueVector(); - OversetFindPointsGSLIB finder(MPI_COMM_WORLD); finder.Setup(*pmesh, color); @@ -431,7 +430,6 @@ int main(int argc, char *argv[]) } // 15. Free the used memory. - finder.FreeData(); delete a; delete fespace; if (order > 0) { delete fec; } diff --git a/miniapps/meshing/makefile b/miniapps/meshing/makefile index a0eb973e66..4df1c96f58 100644 --- a/miniapps/meshing/makefile +++ b/miniapps/meshing/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/meshing/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) @@ -39,8 +35,7 @@ COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common # If MFEM_SHARED is set, add the ../common rpath COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\ - $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\ - $(MFEM_BUILD_DIR)/miniapps/common)) + $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common)) .SUFFIXES: .SUFFIXES: .o .cpp .mk diff --git a/miniapps/meshing/mesh-explorer.cpp b/miniapps/meshing/mesh-explorer.cpp index 73a542b85f..83599f1f50 100644 --- a/miniapps/meshing/mesh-explorer.cpp +++ b/miniapps/meshing/mesh-explorer.cpp @@ -418,8 +418,9 @@ int main (int argc, char *argv[]) "b) Mesh::UniformRefinement() (bisection for tet meshes)\n" "u) uniform refinement with a factor\n" "g) non-uniform refinement (Gauss-Lobatto) with a factor\n" - "n) NURBS refinement (uniform or by formula) with a factor\n" - "c) NURBS coarsening (uniform or by formula) with a factor\n" + "n) NURBS refinement with factors\n" + "p) NURBS NC-patch refinement with factors\n" + "c) NURBS coarsening with a factor\n" "l) refine locally using the region() function\n" "r) random refinement with a probability\n" "--> " << flush; @@ -464,19 +465,38 @@ int main (int argc, char *argv[]) if (ref_factor <= 1 || ref_factor > 32) { break; } char input_tol = 'n'; - cout << "enter NURBS tolerance? [y/n] ---> " << flush; + cout << "enter NURBS tolerance? [y/n] --> " << flush; cin >> input_tol; real_t tol = 1.0e-12; // Default value if (input_tol == 'y') { - cout << "enter NURBS tolerance ---> " << flush; + cout << "enter NURBS tolerance --> " << flush; cin >> tol; } mesh->NURBSUniformRefinement(ref_factors, tol); break; } + case 'p': + { + int ref_factor; + cout << "enter default refinement factor --> " << flush; + cin >> ref_factor; + if (ref_factor <= 1 || ref_factor > 32) { break; } + + cout << "enter knot vector refinement factor filename? [y/n] ---> " << flush; + char input_kvf = 'n'; + cin >> input_kvf; + std::string kvf; + if (input_kvf == 'y') + { + cout << "enter filename ---> " << flush; + cin >> kvf; + } + mesh->RefineNURBSWithKVFactors(ref_factor, kvf); + break; + } case 'c': { cout << "enter coarsening factor --> " << flush; @@ -485,13 +505,13 @@ int main (int argc, char *argv[]) if (coarsen_factor <= 1 || coarsen_factor > 32) { break; } char input_tol = 'n'; - cout << "enter NURBS tolerance? [y/n] ---> " << flush; + cout << "enter NURBS tolerance? [y/n] --> " << flush; cin >> input_tol; real_t tol = 1.0e-12; // Default value if (input_tol == 'y') { - cout << "enter NURBS tolerance ---> " << flush; + cout << "enter NURBS tolerance --> " << flush; cin >> tol; } diff --git a/miniapps/meshing/mesh-optimizer.cpp b/miniapps/meshing/mesh-optimizer.cpp index b9da0e66e5..7fc10fbab9 100644 --- a/miniapps/meshing/mesh-optimizer.cpp +++ b/miniapps/meshing/mesh-optimizer.cpp @@ -105,7 +105,7 @@ // 2D untangling: // mesh-optimizer -m jagged.mesh -o 2 -mid 22 -tid 1 -ni 50 -li 50 -qo 4 -fd -vl 1 // 2D untangling with shifted barrier metric: -// mesh-optimizer -m jagged.mesh -o 2 -mid 4 -tid 1 -ni 50 -qo 4 -fd -vl 1 -btype 1 +// mesh-optimizer -m jagged.mesh -o 2 -mid 4 -tid 1 -ni 50 -qo 4 -vl 1 -btype 1 // 3D untangling (the mesh is in the mfem/data GitHub repository): // * mesh-optimizer -m ../../../mfem_data/cube-holes-inv.mesh -o 3 -mid 313 -tid 1 -rtol 1e-5 -li 50 -qo 4 -fd -vl 1 diff --git a/miniapps/meshing/phpref.cpp b/miniapps/meshing/phpref.cpp index 1389c27e5f..42b3d77629 100644 --- a/miniapps/meshing/phpref.cpp +++ b/miniapps/meshing/phpref.cpp @@ -4,6 +4,7 @@ // // Sample runs: mpirun -np 4 phpref -dim 2 -n 1000 // mpirun -np 8 phpref -dim 3 -n 200 +// mpirun -np 8 phpref -dim 3 -n 20 --anisotropic --fixed-order // // Description: This example demonstrates h- and p-refinement in a parallel // finite element discretization of the Poisson problem (cf. ex1p) @@ -49,6 +50,8 @@ int main(int argc, char *argv[]) bool visualization = true; int numIter = 0; int dim = 2; + bool anisotropic = false; + bool fixedOrder = false; bool deterministic = true; bool projectSolution = false; @@ -63,6 +66,12 @@ int main(int argc, char *argv[]) "Enable or disable GLVis visualization."); args.AddOption(&numIter, "-n", "--num-iter", "Number of hp-ref iterations"); args.AddOption(&dim, "-dim", "--dim", "Mesh dimension (2 or 3)"); + args.AddOption(&anisotropic, "-aniso", "--anisotropic", "-iso", + "--isotropic", + "Whether to use anisotropic refinements"); + args.AddOption(&fixedOrder, "-fo", "--fixed-order", "-vo", + "--variable-order", + "Whether to fix the finite element order on all elements"); args.AddOption(&deterministic, "-det", "--deterministic", "-not-det", "--not-deterministic", "Use deterministic random refinements"); @@ -84,6 +93,9 @@ int main(int argc, char *argv[]) args.PrintOptions(cout); } + MFEM_VERIFY(!anisotropic || fixedOrder, + "Variable-order is not supported with anisotropic refinement"); + // 3. Enable hardware devices such as GPUs, and programming models such as // CUDA, OCCA, RAJA and OpenMP based on command line options. Device device(device_config); @@ -157,15 +169,23 @@ int main(int argc, char *argv[]) const int r2 = deterministic ? DetRand(seed) : rand(); const int elem = r1 % pmesh.GetNE(); int hp = r2 % 2; + char htype = 7; MPI_Bcast(&hp, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (fixedOrder) { hp = 0; } // Only perform h-refinement + if (anisotropic) + { + const int r3 = deterministic ? DetRand(seed) : rand(); + htype = (r3 % 7) + 1; + } + if (myid == 0) cout << "hp-refinement iteration " << iter << ": " - << hp_char[hp] << "-refinement" << endl; + << hp_char[hp] << "-refinement\n"; if (hp == 1) { - // p-ref + // p-refinement Array refs; refs.Append(pRefinement(elem, 1)); // Increase the element order by 1 fespace.PRefineAndUpdate(refs); @@ -173,9 +193,23 @@ int main(int argc, char *argv[]) } else { - // h-ref + // h-refinement Array refs; - refs.Append(Refinement(elem)); + refs.Append(Refinement(elem, htype)); + if (anisotropic) + { + std::set conflicts; // Indices in refs of conflicting elements + const bool conflict = pmesh.AnisotropicConflict(refs, conflicts); + if (conflict) + { + if (myid == 0) + cout << "Anisotropic conflict on iteration " << iter + << ", retrying\n"; + iter--; + continue; + } + } + pmesh.GeneralRefinement(refs); fespace.Update(false); numH++; diff --git a/miniapps/meshing/pmesh-optimizer.cpp b/miniapps/meshing/pmesh-optimizer.cpp index b9bee04091..4f234192fb 100644 --- a/miniapps/meshing/pmesh-optimizer.cpp +++ b/miniapps/meshing/pmesh-optimizer.cpp @@ -107,7 +107,7 @@ // 2D untangling: // mpirun -np 4 pmesh-optimizer -m jagged.mesh -o 2 -mid 22 -tid 1 -ni 50 -li 50 -qo 4 -fd -vl 1 // 2D untangling with shifted barrier metric: -// mpirun -np 4 pmesh-optimizer -m jagged.mesh -o 2 -mid 4 -tid 1 -ni 50 -qo 4 -fd -vl 1 -btype 1 +// mpirun -np 4 pmesh-optimizer -m jagged.mesh -o 2 -mid 4 -tid 1 -ni 50 -qo 4 -vl 1 -btype 1 // 3D untangling (the mesh is in the mfem/data GitHub repository): // * mpirun -np 4 pmesh-optimizer -m ../../../mfem_data/cube-holes-inv.mesh -o 3 -mid 313 -tid 1 -rtol 1e-5 -li 50 -qo 4 -fd -vl 1 // Shape optimization for a Kershaw transformed mesh using partial assembly: diff --git a/miniapps/mtop/makefile b/miniapps/mtop/makefile index dc7d1cb8e8..c55213e771 100644 --- a/miniapps/mtop/makefile +++ b/miniapps/mtop/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/mtop/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) diff --git a/miniapps/navier/navier_cht.cpp b/miniapps/navier/navier_cht.cpp index a98e51c947..acb1803b1f 100644 --- a/miniapps/navier/navier_cht.cpp +++ b/miniapps/navier/navier_cht.cpp @@ -461,7 +461,7 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const Kmat.Mult(u, z); z.Neg(); // z = -z - K->EliminateVDofsInRHS(ess_tdof_list, u, z); + K->ParallelEliminateTDofsInRHS(ess_tdof_list, u, z); M_solver.Mult(z, du_dt); du_dt.Print(); @@ -483,7 +483,7 @@ void ConductionOperator::ImplicitSolve(const real_t dt, MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt Kmat.Mult(u, z); z.Neg(); - K->EliminateVDofsInRHS(ess_tdof_list, u, z); + K->ParallelEliminateTDofsInRHS(ess_tdof_list, u, z); T_solver.Mult(z, du_dt); du_dt.SetSubVector(ess_tdof_list, 0.0); diff --git a/miniapps/shifted/distance.cpp b/miniapps/shifted/distance.cpp index e5f43d74b3..8af4fb3db0 100644 --- a/miniapps/shifted/distance.cpp +++ b/miniapps/shifted/distance.cpp @@ -87,6 +87,8 @@ // // Problem 4: level set: Union of doughnut and swiss cheese shapes // mpirun -np 4 distance -m ../../data/inline-hex.mesh -rs 3 -o 2 -t 1.0 -p 4 +// Problem 5: point source in mfem mesh. +// mpirun -np 4 distance -m ../../data/mfem.mesh -p 5 -rs 3 -t 300.0 #include #include @@ -233,7 +235,8 @@ int main(int argc, char *argv[]) "1: Circle / sphere level set in 2D / 3D\n\t" "2: 2D sine-looking level set\n\t" "3: Gyroid level set in 2D or 3D\n\t" - "4: Combo of a doughnut and swiss cheese shapes in 3D."); + "4: Combo of a doughnut and swiss cheese shapes in 3D.\n\t" + "5: Point source in MFEM mesh."); args.AddOption(&rs_levels, "-rs", "--refine-serial", "Number of times to refine the mesh uniformly in serial."); args.AddOption(&order, "-o", "--order", @@ -299,6 +302,11 @@ int main(int argc, char *argv[]) ls_coeff = new FunctionCoefficient(doughnut_cheese); smooth_steps = 0; } + else if (problem == 5) + { + ls_coeff = new DeltaCoefficient(0.0, 0.0, 1000.0); + smooth_steps = 0; + } else { MFEM_ABORT("Unrecognized -problem option."); } const real_t dx = AvgElementSize(pmesh); @@ -306,7 +314,7 @@ int main(int argc, char *argv[]) if (solver_type == 0) { auto ds = new HeatDistanceSolver(t_param * dx * dx); - if (problem == 0) + if (problem == 0 || problem == 5) { ds->transform = false; } @@ -334,7 +342,7 @@ int main(int argc, char *argv[]) // Smooth-out Gibbs oscillations from the input level set. The smoothing // parameter here is specified to be mesh dependent with length scale dx. ParGridFunction filt_gf(&pfes_s); - if (problem != 0) + if (problem != 0 && problem != 5) { real_t filter_weight = dx; // The normalization-based solver needs a more diffused input. diff --git a/miniapps/shifted/makefile b/miniapps/shifted/makefile index b76eb5b2cf..3b81e891d6 100644 --- a/miniapps/shifted/makefile +++ b/miniapps/shifted/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/shifted/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) @@ -46,8 +42,7 @@ COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common # If MFEM_SHARED is set, add the ../common rpath COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\ - $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\ - $(MFEM_BUILD_DIR)/miniapps/common)) + $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common)) .SUFFIXES: .SUFFIXES: .o .cpp .mk diff --git a/miniapps/solvers/lor_solvers.cpp b/miniapps/solvers/lor_solvers.cpp index 50f9c148cf..e2d5eb10b7 100644 --- a/miniapps/solvers/lor_solvers.cpp +++ b/miniapps/solvers/lor_solvers.cpp @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) else if (string(fe) == "l") { L2 = true; } else { MFEM_ABORT("Bad FE type. Must be 'h', 'n', 'r', or 'l'."); } - real_t kappa = (order+1)*(order+1); // Penalty used for DG discretizations + real_t kappa = 10*(order+1)*(order+1); // Penalty used for DG discretizations Mesh mesh(mesh_file, 1, 1); const int dim = mesh.Dimension(); @@ -156,7 +156,7 @@ int main(int argc, char *argv[]) } // Partial assembly not currently supported for DG or for surface meshes with // vector finite elements (ND or RT). - if (!L2 && (H1 || sdim == dim)) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); } + if (H1 || sdim == dim) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); } a.Assemble(); LinearForm b(&fes); @@ -167,6 +167,7 @@ int main(int argc, char *argv[]) // DG boundary conditions are enforced weakly with this integrator. b.AddBdrFaceIntegrator(new DGDirichletLFIntegrator(u_coeff, -1.0, kappa)); } + if (H1) { b.UseFastAssembly(true); } b.Assemble(); GridFunction x(&fes); diff --git a/miniapps/solvers/plor_solvers.cpp b/miniapps/solvers/plor_solvers.cpp index 090a587e6f..8eb663a52c 100644 --- a/miniapps/solvers/plor_solvers.cpp +++ b/miniapps/solvers/plor_solvers.cpp @@ -112,7 +112,7 @@ int main(int argc, char *argv[]) else if (string(fe) == "l") { L2 = true; } else { MFEM_ABORT("Bad FE type. Must be 'h', 'n', 'r', or 'l'."); } - real_t kappa = (order+1)*(order+1); // Penalty used for DG discretizations + real_t kappa = 10*(order+1)*(order+1); // Penalty used for DG discretizations Mesh serial_mesh(mesh_file, 1, 1); const int dim = serial_mesh.Dimension(); @@ -166,7 +166,7 @@ int main(int argc, char *argv[]) } // Partial assembly not currently supported for DG or for surface meshes with // vector finite elements (ND or RT). - if (!L2 && (H1 || sdim == dim)) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); } + if (H1 || sdim == dim) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); } a.Assemble(); ParLinearForm b(&fes); @@ -177,6 +177,7 @@ int main(int argc, char *argv[]) // DG boundary conditions are enforced weakly with this integrator. b.AddBdrFaceIntegrator(new DGDirichletLFIntegrator(u_coeff, -1.0, kappa)); } + if (H1) { b.UseFastAssembly(true); } b.Assemble(); ParGridFunction x(&fes); diff --git a/miniapps/spde/makefile b/miniapps/spde/makefile index 3be99d1652..8606b1773a 100644 --- a/miniapps/spde/makefile +++ b/miniapps/spde/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/spde/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) diff --git a/miniapps/tools/display-basis.cpp b/miniapps/tools/display-basis.cpp index 34537cf1a0..c7c85aac20 100644 --- a/miniapps/tools/display-basis.cpp +++ b/miniapps/tools/display-basis.cpp @@ -436,7 +436,7 @@ int main(int argc, char *argv[]) cout << endl; cout << "enter new basis function order --> " << flush; cin >> oInt; - if ( oInt >= oMin && oInt <= (oMax>=0)?oMax:oInt ) + if ( oInt >= oMin && oInt <= ((oMax>=0)?oMax:oInt) ) { bOrder = oInt; print_char = true; diff --git a/miniapps/tools/makefile b/miniapps/tools/makefile index 3c3e8a063f..f367fd803c 100644 --- a/miniapps/tools/makefile +++ b/miniapps/tools/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/tools/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) @@ -44,8 +40,7 @@ COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common # If MFEM_SHARED is set, add the ../common rpath COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\ - $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\ - $(MFEM_BUILD_DIR)/miniapps/common)) + $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common)) all: $(MINIAPPS) diff --git a/miniapps/toys/makefile b/miniapps/toys/makefile index f469472a77..257591cca0 100644 --- a/miniapps/toys/makefile +++ b/miniapps/toys/makefile @@ -17,10 +17,6 @@ SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/toys/,) CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\ $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk)) -# Include defaults.mk to get XLINKER -DEFAULTS_MK = $(MFEM_DIR)/config/defaults.mk -include $(DEFAULTS_MK) - MFEM_LIB_FILE = mfem_is_not_built -include $(CONFIG_MK) @@ -41,8 +37,7 @@ COMMON_LIB = -L$(MFEM_BUILD_DIR)/miniapps/common -lmfem-common # If MFEM_SHARED is set, add the ../common rpath COMMON_LIB += $(if $(MFEM_SHARED:YES=),,\ - $(if $(MFEM_USE_CUDA:YES=),$(CXX_XLINKER),$(CUDA_XLINKER))-rpath,$(abspath\ - $(MFEM_BUILD_DIR)/miniapps/common)) + $(MFEM_XLINKER)-rpath,$(abspath $(MFEM_BUILD_DIR)/miniapps/common)) all: $(MINIAPPS) diff --git a/tests/gitlab/README.md b/tests/gitlab/README.md index bccdcc3c90..d25a781d28 100644 --- a/tests/gitlab/README.md +++ b/tests/gitlab/README.md @@ -7,8 +7,18 @@ https://mfem.org +FIXME: this file needs to be updated + This directory contains utility scripts related to GitLab testing at LLNL. +* `build_and_test_setup` is used in CI to setup certain directories before + running the tests. + + This script was designed to be used in CI context. It is easier for the sake + of reproducibility to clone MFEM `data` and `autotest` repository manually + next to your MFEM repository, _unless you are using the inline reproducer + provided in CI, in which case the required variables will be set for you._ + * `build_and_test` is used in CI to build TPLs (dependencies) and MFEM and to perform testing. diff --git a/tests/gitlab/build_and_test b/tests/gitlab/build_and_test index 683b4954e6..6811f380ad 100755 --- a/tests/gitlab/build_and_test +++ b/tests/gitlab/build_and_test @@ -11,6 +11,12 @@ # terms of the BSD-3 license. We welcome feedback and contributions, see file # CONTRIBUTING.md for details. +# Initialize modules for users not using bash as a default shell +if test -e /usr/share/lmod/lmod/init/bash +then + . /usr/share/lmod/lmod/init/bash +fi + set -o errexit set -o nounset @@ -60,15 +66,40 @@ function usage() } project_dir="$(pwd)" +hostname="$(hostname)" -mode="" spec="" +mode="" data_dir="" with_data=false sys_type=${SYS_TYPE:-""} - threads=${THREADS:-""} +module_list=${MODULE_LIST:-""} +job_unique_id=${CI_JOB_ID:-""} +use_dev_shm=${USE_DEV_SHM:-true} +spack_debug=${SPACK_DEBUG:-false} +debug_mode=${DEBUG_MODE:-false} + +# CI_REGISTRY_USER, CI_REGISTRY_IMAGE, and CI_JOB_TOKEN are automatically set by +# Gitlab, see https://docs.gitlab.com/ee/ci/variables/predefined_variables.html. +# The Spack build cache files can be viewed from the Gitlab web interface under +# "Deploy" -> "Container Registry". +# +# Since the CI_ are only set in CI, we provide sensible defaults for +# usage outside CI context (e.g. to reproduce a CI scenario locally). +# +# REGISTRY_TOKEN when set allows to provide our own personal access token to +# the CI registry. Be sure to set the token with at least read access to the +# registry. It is optional outside CI but will speed up the build if many +# dependencies are needed. +# +registry_token=${REGISTRY_TOKEN:-""} +ci_registry_user=${CI_REGISTRY_USER:-"${USER}"} +ci_registry_image=${CI_REGISTRY_IMAGE:-"czregistry.llnl.gov:5050/mfem/mfem-autotest"} +ci_registry_token=${CI_JOB_TOKEN:-"${registry_token}"} + +spec_tab=() # Options while [[ $# -gt 0 ]] @@ -77,9 +108,19 @@ do case $key in --spec) - spec="$2" - shift # past argument - shift # past value + shift + arg="${1}" + if [[ "${arg}" =~ ^\" ]]; then + arg="${arg#\"}" + while [[ ! "${arg}" =~ [^\\]\"$ ]]; do + spec_tab+=("${arg}") + shift + arg="${1}" + done + arg="${arg%\"}" + fi + spec_tab+=("${arg}") + shift ;; --deps-only|--build-only|--test-only) mode="$key" @@ -99,48 +140,133 @@ do exit 0; ;; *) # unknown option - echo "ERROR: option $key is unknown" + echo "[Error]: option $key is unknown" exit 1; ;; esac done +spec="${spec_tab[@]}" + +timed_message () +{ + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "~ $(date --rfc-3339=seconds) ~ ${1}" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +} + +at_exit() +{ + if [[ "${debug_mode}" != "true" ]]; then + if [[ "${#exit_rm_list[@]}" != 0 ]]; then + echo ${script_name}:at_exit: rm -rf "${exit_rm_list[@]}" + rm -rf "${exit_rm_list[@]}" + fi + if [[ "${exit_distclean}" == 1 ]]; then + echo ${script_name}:at_exit: \ + make -C "${project_dir}" distclean "> /dev/null 2>&1" + make -C "${project_dir}" distclean > /dev/null 2>&1 + fi + fi +} + +if [[ ${debug_mode} == true ]] +then + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "~~~~~ Debug mode:" + echo "~~~~~ - Spack debug mode." + echo "~~~~~ - Deactivated shared memory." + echo "~~~~~ - Do not push to buildcache." + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + use_dev_shm=false + spack_debug=true +fi + +if [[ -n ${module_list} ]] +then + timed_message "Modules to load: ${module_list}" + module load ${module_list} +fi + +prefix="" + +if [[ -d /dev/shm && ${use_dev_shm} == true ]] +then + prefix="/dev/shm/${hostname}" + if [[ -z ${job_unique_id} ]]; then + job_unique_id=manual_job_$(date +%s) + while [[ -d ${prefix}-${job_unique_id} ]] ; do + sleep 1 + job_unique_id=manual_job_$(date +%s) + done + fi + + prefix="${prefix}-${job_unique_id}" +else + # We set the prefix in the parent directory so that spack dependencies are + # not installed inside the source tree. + prefix="${project_dir}/../spack-and-build-root" +fi + +echo "Creating directory ${prefix}" +echo "project_dir: ${project_dir}" + +mkdir -p ${prefix} + +# List of files/directories to remove at exit; more entries are added below. +# This list will be removed even if there's an error. In debug mode, this +# cleanup is not performed. +exit_rm_list=("${prefix}") +exit_distclean= +trap 'at_exit' EXIT HUP INT QUIT ABRT KILL PIPE TERM XCPU + +# Prefix tree: +# +# ├── spack +# ├── spack_env +# └── spack-user-cache + +spack_cmd="${prefix}/spack/bin/spack" +spack_env_path="${prefix}/spack_env" +command -v python3 > /dev/null && python_cmd=python3 || python_cmd=python +uberenv_cmd="${python_cmd} tests/uberenv/uberenv.py" +if [[ ${spack_debug} == true ]] +then + spack_cmd="${spack_cmd} --debug --stacktrace" + uberenv_cmd="${uberenv_cmd} --spack-debug" +fi + # Dependencies if [[ "${mode}" != "--build-only" && "${mode}" != "--test-only" ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Building Dependencies" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Building dependencies" if [[ -z ${spec} ]] then - echo "--spec is mandatory to build dependencies, aborting..." + echo "[Error]: SPEC is undefined, aborting..." exit 1 fi - upstream_opt="" - if [[ ${sys_type} != "" ]] - then - upstream_opt="--upstream=/usr/workspace/mfem/mfem-dependencies/install" - fi + prefix_opt="--prefix=${prefix}" - prefix_opt="" - - # Do not run on /dev/shm if only installing dependencies: they will be lost - # otherwise. - if [[ -d "/dev/shm" && "${mode}" != "--deps-only" ]] - then - prefix="/dev/shm/${USER}_${CI_PIPELINE_ID:-"NONE"}_${RANDOM}" - mkdir -p ${prefix} - # clean up ${prefix} at exit: - trap 'rm -rf "${prefix}"' EXIT - prefix_opt="--prefix=${prefix}" - fi echo ${spec} > spec.txt - echo "Fetching uberenv." + # We force Spack to put all generated files (cache and configuration of + # all sorts) in a unique location so that there can be no collision + # with existing or concurrent Spack. + spack_user_cache="${prefix}/spack-user-cache" + export SPACK_DISABLE_LOCAL_CONFIG="" + export SPACK_USER_CACHE_PATH="${spack_user_cache}" + mkdir -p ${spack_user_cache} + + echo "Fetching uberenv ..." + # FIXME: the messages from the next clone appear out-of-place, at the end of + # the logs on Gitlab. (On Lassen the messages are where expected.) tests/gitlab/get_mfem_uberenv || { echo "Error fetching Uberenv"; exit 1; } + # Remove "${project_dir}/tests/uberenv" at exit (even if there's an error) + exit_rm_list=("${exit_rm_list[@]}" "${project_dir}/tests/uberenv") + echo "Removing existing configuration" make distclean @@ -149,10 +275,38 @@ then config/defaults.mk.new mv -f config/defaults.mk.new config/defaults.mk - echo "Installing dependencies." - command -v python > /dev/null && python_cmd=python || python_cmd=python3 - $python_cmd tests/uberenv/uberenv.py --spec="${spec}" "${upstream_opt}" \ - "${prefix_opt}" + # generate cmake cache file with uberenv and radiuss spack package + timed_message "Spack setup and environment" + ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt} + + if [[ -n ${ci_registry_token} ]] + then + timed_message "GitLab registry as Spack Buildcache" + ${spack_cmd} -D ${spack_env_path} mirror add --unsigned \ + --oci-username ${ci_registry_user} \ + --oci-password ${ci_registry_token} \ + gitlab_ci oci://${ci_registry_image} + fi + + timed_message "Spack build of dependencies" + ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt} || \ + { + # Save Spack logs + cd ${SPACK_USER_CACHE_PATH} + log_archive="${project_dir}/spack-logs-${hostname}-${CI_JOB_ID}.tgz" + tar zvcf "${log_archive}" stage/*/*/spack*.txt + timed_message "Spack build failed! See log archive: ${log_archive}" + exit 1 + } + + if [[ -n ${ci_registry_token} && ${debug_mode} == false ]] + then + timed_message "Push dependencies to buildcache" + ${spack_cmd} -D ${spack_env_path} buildcache push \ + --only dependencies gitlab_ci + fi + + timed_message "Dependencies built" # Make sure that a configuration was generated by spack (part 1). cp config/config.mk config/spack-config.mk @@ -170,10 +324,10 @@ then cp config/spack-config.mk config/config.mk cp config/spack_config.hpp config/_config.hpp else - echo "No result for at least one of" - echo " ${project_dir}/config/spack-config.mk" - echo " ${project_dir}/config/spack_config.hpp" - echo "ERROR: Spack generated configuration not found." + echo "[Error]: No result for at least one of" + echo "[Error]: ${project_dir}/config/spack-config.mk" + echo "[Error]: ${project_dir}/config/spack_config.hpp" + echo "[Error]: Spack generated configuration not found." exit 1 fi @@ -188,10 +342,10 @@ then # project parent dir. if [[ -e "../data" ]]; then if [[ -L "../data" ]]; then - echo "'../data' link already exists. Deleting." + echo "[Information]: '../data' link already exists. Deleting." rm "../data" else - echo "Error: '../data' already exists and it's NOT a link" + echo "[Error]: '../data' already exists and it's NOT a link" exit 1 fi fi @@ -204,7 +358,7 @@ then if [[ "$with_data" == "true" && ! -d "../data" ]] then - echo "ERROR: '$data_dir' is not a directory while asking for --data" + echo "[Error]: '$data_dir' is not a directory while asking for --data" exit 1 fi fi @@ -216,42 +370,45 @@ then echo "~ Project Dir: ${project_dir}" echo "~ Data Dir: ${data_dir}" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ MFEM configuration" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "MFEM Configuration" make info - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Building MFEM" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Building MFEM" + + # If the build is terminated, e.g. job timeout, run 'make distclean' + exit_distclean=1 make all -j ${threads} + + exit_distclean= fi # Test if [[ "${mode}" != "--deps-only" && "${mode}" != "--build-only" ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Testing MFEM" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Testing MFEM" + + # If the build is terminated, e.g. job timeout, run 'make distclean' + exit_distclean=1 make test test_status=$? - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Cleaning MFEM" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + timed_message "Cleaning MFEM" if make distclean > distclean.log 2>&1; then - echo "[make distclean]: OK" + echo "[Information]: make distclean OK" else cat distclean.log echo - echo "[make distclean]: FAILED (see above)" + echo "[Information]: make distclean FAILED (see above)" fi rm -f distclean.log + exit_distclean= + exit $test_status fi diff --git a/tests/gitlab/build_and_test_setup b/tests/gitlab/build_and_test_setup new file mode 100755 index 0000000000..06a2897527 --- /dev/null +++ b/tests/gitlab/build_and_test_setup @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +# at the Lawrence Livermore National Laboratory. All Rights reserved. See files +# LICENSE and NOTICE for details. LLNL-CODE-806117. +# +# This file is part of the MFEM library. For more information and source code +# availability visit https://mfem.org. +# +# MFEM is free software; you can redistribute it and/or modify it under the +# terms of the BSD-3 license. We welcome feedback and contributions, see file +# CONTRIBUTING.md for details. + +function info_msg () +{ + echo "[Information:] ${1}" +} + +function error_msg () +{ + echo "[Error:] ${1}" +} + +# Perform a clone while holding a lock file to prevent concurrency on +# the destination. +# Usage: +# locked_clone +function locked_clone () +{ + if ! command -v flock + then + error_msg "Required command 'flock' not found" + exit 1 + fi + + info_msg "Will clone ${1} into ${2}" + + ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..." + # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the + # try every 5 seconds; we may want to add a counter for the number of + # retries to interrupt a potential infinite loop + while ! flock -n 9; do sleep 5; done + date; info_msg "Acquired lock on '${PWD}/${2}.lock'" + # clone/update the autotest repo while holding the file lock on + # 'autotest.lock' + err=0 + if [[ ! -d "${2}" ]]; then + git clone ${1} ${2} + else + cd ${2} && git pull && cd .. + fi || err=1 + # sleep for a period to allow NFS to propagate the above changes; + # clearly, there is no guarantee that other NFS clients will see the + # changes even after the timeout + sleep 10 + exit $err + ) 9> ${2}.lock +} + +# Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml' +info_msg "MFEM_DATA_REPO is ${MFEM_DATA_REPO}" +info_msg "SHARED_REPOS_DIR is ${SHARED_REPOS_DIR}" + +mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR} +locked_clone ${MFEM_DATA_REPO} mfem-data + +# Setup ${AUTOTEST_ROOT}/autotest: +info_msg "AUTOTEST_REPO is ${AUTOTEST_REPO}" +info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}" + +mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT} +locked_clone ${AUTOTEST_REPO} autotest diff --git a/tests/gitlab/generate_spack_upstream b/tests/gitlab/generate_spack_upstream deleted file mode 100755 index 04784112e0..0000000000 --- a/tests/gitlab/generate_spack_upstream +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced -# at the Lawrence Livermore National Laboratory. All Rights reserved. See files -# LICENSE and NOTICE for details. LLNL-CODE-806117. -# -# This file is part of the MFEM library. For more information and source code -# availability visit https://mfem.org. -# -# MFEM is free software; you can redistribute it and/or modify it under the -# terms of the BSD-3 license. We welcome feedback and contributions, see file -# CONTRIBUTING.md for details. - -# This script is meant be launched in the root directory of MFEM to compile all -# the dependencies of MFEM for all the specs in CI. By doing so, we obtain a -# Spack instance that can be used as an upstream. -# Note: needs mfem-uberenv in MFEM repo (run tests/gitlab/get_mfem_uberenv) - -# This patch applies shared permissions to Spack installations. -cd tests/uberenv -git apply upstream-permission.patch -cd - - -# Get the hostname -hostid=${HOSTNAME//[[:digit:]]/} - -# call uberenv for all specs in CI -git grep -e "^[^#]" .gitlab/$hostid* | grep "SPEC" \ - | cut -d' ' -f6- \ - | sed 's/"//g' \ - | while read -r line; do - python ./tests/uberenv/uberenv.py --spec="$line" -done - -# We revert the patch to leave the repo as found. -cd tests/uberenv -git apply -R upstream-permission.patch -cd - diff --git a/tests/gitlab/get_mfem_uberenv b/tests/gitlab/get_mfem_uberenv index 99f2aae870..07c8918090 100755 --- a/tests/gitlab/get_mfem_uberenv +++ b/tests/gitlab/get_mfem_uberenv @@ -18,7 +18,7 @@ set -o errexit set -o nounset uberenv_url="https://github.com/mfem/mfem-uberenv.git" -uberenv_ref="fe5fa88876b29ff03177d44a5bd3e09c84ccdcbf" +uberenv_ref="a9bd720f83cf1f8afa26b4e69737fcd4a6d462f9" [[ ! -d tests/uberenv ]] && git clone ${uberenv_url} tests/uberenv cd tests/uberenv diff --git a/tests/gitlab/reproduce-ci-jobs-interactively.md b/tests/gitlab/reproduce-ci-jobs-interactively.md index de3d6bcd9e..bad118f165 100644 --- a/tests/gitlab/reproduce-ci-jobs-interactively.md +++ b/tests/gitlab/reproduce-ci-jobs-interactively.md @@ -1,3 +1,6 @@ + +FIXME: this file needs to be updated + # HowTo: Reproduce CI jobs interactively. We rely on Spack, driven by uberenv, to build MFEM dependencies automatically @@ -26,6 +29,48 @@ Those modes are essentially the same, but we emphasize building the dependencies as a first isolated step because it makes it clear what is happening and how to use this workflow. +## Shortcut: + +To help developers reproduce jobs from the CI, a reproducer script is printed +in each job that will allow for fast and accurate reproduction of the same +scenario as in the CI job. Below is an example extracted for a CI job log: + +```bash +working_dir="/usr/workspace/${USER}/mfem/2405156-$(date +%s)" +mkdir -p ${working_dir} && cd ${working_dir} +git clone https://github.com/MFEM/mfem.git --single-branch --depth=1 +cd mfem +git fetch origin --depth=1 4868222660f03e15ebf7a6daca90800b64baf69d +git checkout 4868222660f03e15ebf7a6daca90800b64baf69d +git submodule update --init --recursive + +# Variables +export SPEC="%gcc@8.3.1 +mpi +cuda cuda_arch=70" + +# Directories +export BUILD_ROOT="${working_dir}" +export SHARED_REPOS_DIR="${BUILD_ROOT}/.." +export MFEM_DATA_DIR="${SHARED_REPOS_DIR}/mfem-data" + +# Repositories +export TPLS_REPO="ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git" +export TESTS_REPO="ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git" +export AUTOTEST_REPO="ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git" +export MFEM_DATA_REPO="https://github.com/mfem/data.git" + +# Setup directories +./tests/gitlab/build_and_test_setup + +# Using the CI build cache is optional and requires a token. Set it like so: +# export REGISTRY_TOKEN="" + +lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "%gcc@8.3.1 +mpi +cuda cuda_arch=70" --data-dir "/usr/workspace/mfem/gitlab-runner/bernede1/repos/mfem-data" --data +``` + +**NOTE** + +The REGISTRY_TOKEN can be set using a GitLab Personal Access Token (PAT) with read access to gitlab container registry. This allows to speed up the local builds by fetching the dependencies library instead of building them. + ## Prerequisite: Retrieve Uberenv ```bash @@ -88,6 +133,11 @@ but still ready to use. ## Build and test MFEM +**NOTE** + +If you need mfem data and/or the autotest repo, you may clone them next to your +mfem repository. + ### Option 1: Without using scripts ```bash diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 33cfdb1dc6..2533011f6e 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -15,16 +15,18 @@ project(mfem-unit-tests NONE) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) # The following list can be updated using (in bash): -# for d in general linalg mesh fem enzyme; do ls -1 $d/*.cpp; done +# for d in dfem general linalg mesh fem enzyme; do ls -1 $d/*.cpp; done set(UNIT_TESTS_SRCS dfem/test_diffusion.cpp dfem/test_divergence.cpp dfem/test_mass.cpp + dfem/test_lvector_interface.cpp general/test_array.cpp - general/test_reduction.cpp + general/test_scan.cpp general/test_arrays_by_name.cpp general/test_error.cpp general/test_mem.cpp + general/test_reduction.cpp general/test_text.cpp general/test_umpire_mem.cpp general/test_zlib.cpp @@ -49,13 +51,15 @@ set(UNIT_TESTS_SRCS linalg/test_ode2.cpp linalg/test_operator.cpp linalg/test_vector.cpp - mesh/test_face_orientations.cpp - mesh/test_geometric_factors.cpp mesh/mesh_test_utils.cpp mesh/test_exodus_reader.cpp + mesh/test_exodus_writer.cpp + mesh/test_face_orientations.cpp mesh/test_fms.cpp + mesh/test_geometric_factors.cpp mesh/test_mesh.cpp mesh/test_ncmesh.cpp + mesh/test_nurbs.cpp mesh/test_periodic_mesh.cpp mesh/test_pmesh.cpp mesh/test_psubmesh.cpp @@ -63,10 +67,10 @@ set(UNIT_TESTS_SRCS mesh/test_vtu.cpp mesh/test_nurbs.cpp mesh/test_exodus_writer.cpp + fem/make_permuted_mesh.cpp fem/test_1d_bilininteg.cpp fem/test_2d_bilininteg.cpp fem/test_3d_bilininteg.cpp - fem/test_assemblediagonalpa.cpp fem/test_assembly_levels.cpp fem/test_bilinearform.cpp fem/test_block_operators.cpp @@ -78,6 +82,7 @@ set(UNIT_TESTS_SRCS fem/test_calcshape.cpp fem/test_calcvshape.cpp fem/test_coefficient.cpp + fem/test_col_lag_der.cpp fem/test_datacollection.cpp fem/test_derefine.cpp fem/test_dgmassinv.cpp @@ -89,15 +94,17 @@ set(UNIT_TESTS_SRCS fem/test_face_elem_trans.cpp fem/test_face_permutation.cpp fem/test_face_restriction.cpp - fem/test_fe.cpp fem/test_fe_compatibility.cpp fem/test_fe_fixed.cpp fem/test_fe_pos.cpp fem/test_fe_symmetry.cpp + fem/test_fe.cpp fem/test_get_value.cpp fem/test_getderivative.cpp fem/test_getgradient.cpp + fem/test_getgradients.cpp fem/test_gslib.cpp + fem/test_hp_transfer.cpp fem/test_intrules.cpp fem/test_intruletypes.cpp fem/test_inversetransform.cpp @@ -105,18 +112,21 @@ set(UNIT_TESTS_SRCS fem/test_lin_interp.cpp fem/test_linear_fes.cpp fem/test_linearform_ext.cpp - fem/test_lor.cpp fem/test_lor_batched.cpp + fem/test_lor_dg.cpp + fem/test_lor.cpp fem/test_nonlinearform.cpp fem/test_operatorjacobismoother.cpp fem/test_oscillation.cpp fem/test_pa_coeff.cpp + fem/test_pa_diagonal.cpp fem/test_pa_grad.cpp fem/test_pa_idinterp.cpp fem/test_pa_kernels.cpp fem/test_pgridfunc_save_serial.cpp - fem/test_project_bdr.cpp + fem/test_poly1d.cpp fem/test_project_bdr_par.cpp + fem/test_project_bdr.cpp fem/test_quadf_coef.cpp fem/test_quadinterpolator.cpp fem/test_quadraturefunc.cpp @@ -144,18 +154,21 @@ set(UNIT_TESTS_SRCS # SERIAL CPU TESTS: unit_tests #----------------------------------------------------------- if (MFEM_USE_CUDA) - set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS} + set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA) endif() if (MFEM_USE_HIP) - set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS} + set_property(SOURCE unit_test_main.cpp ${UNIT_TESTS_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE) endif() +add_library(unit_tests_srcs OBJECT ${UNIT_TESTS_SRCS}) +target_link_libraries(unit_tests_srcs PUBLIC mfem) + # All serial non-device unit tests are built into a single executable, # 'unit_tests'. -mfem_add_executable(unit_tests unit_test_main.cpp ${UNIT_TESTS_SRCS}) -target_link_libraries(unit_tests mfem) +mfem_add_executable(unit_tests unit_test_main.cpp) +target_link_libraries(unit_tests unit_tests_srcs) add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} unit_tests) # Unit tests need the ../../data directory. add_dependencies(unit_tests copy_data) @@ -175,7 +188,7 @@ COMMAND ${CMAKE_COMMAND} -E copy_directory # make unit_tests # ctest -R unit_tests [-V] if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME unit_tests COMMAND unit_tests) + add_test(NAME unit_tests COMMAND unit_tests) endif() #----------------------------------------------------------- @@ -183,16 +196,16 @@ endif() #----------------------------------------------------------- # Create CUDA executable and test if (MFEM_USE_CUDA) - # gpu_unit_tests - set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp) - set_property(SOURCE ${GPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA) - mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS}) - target_link_libraries(gpu_unit_tests mfem) - add_dependencies(gpu_unit_tests copy_data) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests) - if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests) - endif() + # gpu_unit_tests + set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp) + set_property(SOURCE ${GPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA) + mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS}) + target_link_libraries(gpu_unit_tests unit_tests_srcs) + add_dependencies(gpu_unit_tests copy_data) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests) + if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE + add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests) + endif() endif() #----------------------------------------------------------- @@ -200,15 +213,15 @@ endif() #----------------------------------------------------------- # Create HIP 'gpu_unit_tests' executable and test if (MFEM_USE_HIP) - # gpu_unit_tests - set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp) - mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS}) - target_link_libraries(gpu_unit_tests mfem) - add_dependencies(gpu_unit_tests copy_data) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests) - if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests) - endif() + # gpu_unit_tests + set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp) + mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS}) + target_link_libraries(gpu_unit_tests unit_tests_srcs) + add_dependencies(gpu_unit_tests copy_data) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests) + if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE + add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests) + endif() endif() #----------------------------------------------------------- @@ -224,7 +237,7 @@ function(add_serial_miniapp_test name test_uvm) set(${NAME}_TESTS_SRCS miniapps/test_${name}.cpp) if (MFEM_USE_CUDA) - set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA) + set_property(SOURCE ${${NAME}_TESTS_SRCS} PROPERTY LANGUAGE CUDA) endif(MFEM_USE_CUDA) mfem_add_executable(${name}_tests_cpu ${${NAME}_TESTS_SRCS}) @@ -244,25 +257,25 @@ function(add_serial_miniapp_test name test_uvm) endif() if (MFEM_USE_CUDA OR MFEM_USE_HIP) - mfem_add_executable(${name}_tests_gpu ${${NAME}_TESTS_SRCS}) - target_compile_definitions(${name}_tests_gpu PUBLIC MFEM_${NAME}_DEVICE="gpu") - target_link_libraries(${name}_tests_gpu mfem) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu) - if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME ${name}_tests_gpu COMMAND ${name}_tests_gpu) - endif() + mfem_add_executable(${name}_tests_gpu ${${NAME}_TESTS_SRCS}) + target_compile_definitions(${name}_tests_gpu PUBLIC MFEM_${NAME}_DEVICE="gpu") + target_link_libraries(${name}_tests_gpu mfem) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu) + if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE + add_test(NAME ${name}_tests_gpu COMMAND ${name}_tests_gpu) + endif() - if (test_uvm) - mfem_add_executable(${name}_tests_gpu_uvm ${${NAME}_TESTS_SRCS}) - target_compile_definitions(${name}_tests_gpu_uvm PUBLIC + if (test_uvm) + mfem_add_executable(${name}_tests_gpu_uvm ${${NAME}_TESTS_SRCS}) + target_compile_definitions(${name}_tests_gpu_uvm PUBLIC MFEM_${NAME}_DEVICE="gpu:uvm") - target_link_libraries(${name}_tests_gpu_uvm mfem) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} + target_link_libraries(${name}_tests_gpu_uvm mfem) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu_uvm) - if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME ${name}_tests_gpu_uvm COMMAND ${name}_tests_gpu_uvm) - endif() - endif() + if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE + add_test(NAME ${name}_tests_gpu_uvm COMMAND ${name}_tests_gpu_uvm) + endif() + endif() endif() endfunction(add_serial_miniapp_test) @@ -277,25 +290,25 @@ add_dependencies(tmop_pa_tests_cpu copy_miniapps_meshing_data) #----------------------------------------------------------- # Add 'ceed_tests' executable and test; add extra tests 'ceed_test_*' if (MFEM_USE_CEED) - set(CEED_TESTS_SRCS + set(CEED_TESTS_SRCS ceed/test_ceed.cpp ceed/test_ceed_main.cpp) - if (MFEM_USE_CUDA) - set_property(SOURCE ${CEED_TESTS_SRCS} PROPERTY LANGUAGE CUDA) - endif(MFEM_USE_CUDA) - mfem_add_executable(ceed_tests ${CEED_TESTS_SRCS}) - target_link_libraries(ceed_tests mfem) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ceed_tests) - # Add CEED tests - add_test(NAME ceed_tests COMMAND ceed_tests) - if (MFEM_USE_CUDA) - add_test(NAME ceed_tests_cuda_ref + if (MFEM_USE_CUDA) + set_property(SOURCE ${CEED_TESTS_SRCS} PROPERTY LANGUAGE CUDA) + endif(MFEM_USE_CUDA) + mfem_add_executable(ceed_tests ${CEED_TESTS_SRCS}) + target_link_libraries(ceed_tests mfem) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ceed_tests) + # Add CEED tests + add_test(NAME ceed_tests COMMAND ceed_tests) + if (MFEM_USE_CUDA) + add_test(NAME ceed_tests_cuda_ref COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/ref) - add_test(NAME ceed_tests_cuda_shared + add_test(NAME ceed_tests_cuda_shared COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/shared) - add_test(NAME ceed_tests_cuda_gen + add_test(NAME ceed_tests_cuda_gen COMMAND ceed_tests --device ceed-cuda:/gpu/cuda/gen) - endif() + endif() endif() #----------------------------------------------------------- @@ -303,54 +316,54 @@ endif() #----------------------------------------------------------- # Define executables and tests if (MFEM_USE_MPI) - # punit_tests - if (MFEM_USE_CUDA) - set_property(SOURCE punit_test_main.cpp PROPERTY LANGUAGE CUDA) - endif() - mfem_add_executable(punit_tests punit_test_main.cpp ${UNIT_TESTS_SRCS}) - target_link_libraries(punit_tests mfem) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} punit_tests) - foreach(np 1 ${MFEM_MPI_NP}) - if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME punit_tests_np=${np} + # punit_tests + if (MFEM_USE_CUDA) + set_property(SOURCE punit_test_main.cpp PROPERTY LANGUAGE CUDA) + endif() + mfem_add_executable(punit_tests punit_test_main.cpp) + target_link_libraries(punit_tests unit_tests_srcs) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} punit_tests) + foreach(np 1 ${MFEM_MPI_NP}) + if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE + add_test(NAME punit_tests_np=${np} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) - endif() - endforeach() - if (MFEM_USE_CUDA) - # pgpu_unit_tests - set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp) - set_property(SOURCE ${PGPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA) - mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS}) - add_dependencies(pgpu_unit_tests copy_data) - target_link_libraries(pgpu_unit_tests mfem) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests) - foreach(np 1 ${MFEM_MPI_NP}) - if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME pgpu_unit_tests_np=${np} + endif() + endforeach() + if (MFEM_USE_CUDA) + # pgpu_unit_tests + set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp) + set_property(SOURCE ${PGPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA) + mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS}) + add_dependencies(pgpu_unit_tests copy_data) + target_link_libraries(pgpu_unit_tests unit_tests_srcs) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests) + foreach(np 1 ${MFEM_MPI_NP}) + if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE + add_test(NAME pgpu_unit_tests_np=${np} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) - endif() - endforeach() + endif() + endforeach() endif() if (MFEM_USE_HIP) - # pgpu_unit_tests - set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp) - mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS}) - add_dependencies(pgpu_unit_tests copy_data) - target_link_libraries(pgpu_unit_tests mfem) - add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests) - foreach(np 1 ${MFEM_MPI_NP}) - if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE - add_test(NAME pgpu_unit_tests_np=${np} + # pgpu_unit_tests + set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp) + mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS}) + add_dependencies(pgpu_unit_tests copy_data) + target_link_libraries(pgpu_unit_tests unit_tests_srcs) + add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests) + foreach(np 1 ${MFEM_MPI_NP}) + if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE + add_test(NAME pgpu_unit_tests_np=${np} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) - endif() - endforeach() - endif() + endif() + endforeach() + endif() endif(MFEM_USE_MPI) #----------------------------------------------------------- @@ -424,8 +437,8 @@ endfunction(add_parallel_miniapp_test) # Additional MPI unit tests if (MFEM_USE_MPI) - add_parallel_miniapp_test(sedov TRUE) - add_parallel_miniapp_test(tmop_pa FALSE) + add_parallel_miniapp_test(sedov TRUE) + add_parallel_miniapp_test(tmop_pa FALSE) endif(MFEM_USE_MPI) #----------------------------------------------------------- @@ -434,10 +447,10 @@ endif(MFEM_USE_MPI) #----------------------------------------------------------- set(DEBUG_DEVICE_SRCS miniapps/test_debug_device.cpp) if (MFEM_USE_CUDA) - set_property(SOURCE ${DEBUG_DEVICE_SRCS} PROPERTY LANGUAGE CUDA) + set_property(SOURCE ${DEBUG_DEVICE_SRCS} PROPERTY LANGUAGE CUDA) endif() if (MFEM_USE_HIP) - set_property(SOURCE ${DEBUG_DEVICE_SRCS} + set_property(SOURCE ${DEBUG_DEVICE_SRCS} PROPERTY HIP_SOURCE_PROPERTY_FORMAT TRUE) endif() mfem_add_executable(debug_device_tests ${DEBUG_DEVICE_SRCS}) diff --git a/tests/unit/dfem/test_diffusion.cpp b/tests/unit/dfem/test_diffusion.cpp index fe4832d071..bf50830af8 100644 --- a/tests/unit/dfem/test_diffusion.cpp +++ b/tests/unit/dfem/test_diffusion.cpp @@ -226,7 +226,7 @@ void DFemDiffusion(const char *filename, int p, const int r) auto dRdU = dop_mf.GetDerivative(U, {&x}, {&rho_coeff_cv, nodes}); pfes.GetRestrictionMatrix()->Mult(x, X); - dop_mf.Mult(X, Z); + dRdU->Mult(X, Z); blf_fa.Mult(x, y); pfes.GetProlongationMatrix()->MultTranspose(y, Y); @@ -255,8 +255,8 @@ void DFemDiffusion(const char *filename, int p, const int r) DOperator dop_mf(vsol, {{Coords, mfes}}, pmesh); const auto mf_vector_diffusion_qf = [] MFEM_HOST_DEVICE (const tensor &dudxi, - const tensor &J, - const real_t &w) + const tensor &J, + const real_t &w) { const auto invJ = inv(J), TinJ = transpose(invJ); return tuple{ (dudxi * invJ) * TinJ * det(J) * w }; diff --git a/tests/unit/dfem/test_lvector_interface.cpp b/tests/unit/dfem/test_lvector_interface.cpp new file mode 100644 index 0000000000..f1125a1ebd --- /dev/null +++ b/tests/unit/dfem/test_lvector_interface.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#include "../unit_tests.hpp" +#include "mfem.hpp" +#include + +#ifdef MFEM_USE_MPI + +using namespace mfem; +using namespace mfem::future; +using mfem::future::tensor; + +constexpr int DIM = 3; + +namespace kernels +{ +struct MFApply +{ + MFEM_HOST_DEVICE inline auto operator()(const tensor &dudxi, + const tensor &J, + const real_t &w) const + { + const auto invJ = inv(J); + return tuple{ (dudxi * invJ) * transpose(invJ) * det(J) * w }; + } +}; +} + +TEST_CASE("DFEM L-Vector interface", "[Parallel][DFEM]") +{ + constexpr int p = 2; // Polynomial order + constexpr int r = 1; + constexpr int q = 2 * p + r; + + const auto filename = GENERATE("../../data/fichera.mesh"); + Mesh smesh(filename); + ParMesh pmesh(MPI_COMM_WORLD, smesh); + MFEM_VERIFY(pmesh.Dimension() == DIM, "Mesh dimension mismatch"); + + pmesh.EnsureNodes(); + auto *nodes = static_cast(pmesh.GetNodes()); + smesh.Clear(); + + Array all_domain_attr; + if (pmesh.attributes.Size() > 0) + { + all_domain_attr.SetSize(pmesh.attributes.Max()); + all_domain_attr = 1; + } + + H1_FECollection fec(p, DIM); + ParFiniteElementSpace pfes(&pmesh, &fec); + ParFiniteElementSpace *mfes = nodes->ParFESpace(); + + const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), q); + + ParGridFunction x(&pfes), y(&pfes), z(&pfes); + Vector X(pfes.GetTrueVSize()), Y(pfes.GetTrueVSize()), Z(pfes.GetTrueVSize()); + + X.Randomize(1); + x.SetFromTrueDofs(X); + + ParBilinearForm blf_fa(&pfes); + blf_fa.AddDomainIntegrator(new DiffusionIntegrator(ir)); + blf_fa.Assemble(); + blf_fa.Finalize(); + + static constexpr int U = 0, Coords = 1; + + const auto solution = std::vector{FieldDescriptor{U, &pfes}}; + DifferentiableOperator dop(solution, {{Coords, mfes}}, pmesh); + + kernels::MFApply mf_apply_qf; + dop.AddDomainIntegrator(mf_apply_qf, + tuple{Gradient{}, Gradient{}, Weight{}}, + tuple{Gradient{}}, *ir, all_domain_attr); + + // Use the L-vector interface to multiply + dop.SetMultLevel(DifferentiableOperator::MultLevel::LVECTOR); + dop.SetParameters({nodes}); + dop.Mult(x, z); + + blf_fa.Mult(x, y); + + z -= y; + REQUIRE(z.Normlinf() == MFEM_Approx(0.0)); +} + +#endif diff --git a/tests/unit/dfem/test_mass.cpp b/tests/unit/dfem/test_mass.cpp index c8cbc69f1d..86200b1364 100644 --- a/tests/unit/dfem/test_mass.cpp +++ b/tests/unit/dfem/test_mass.cpp @@ -18,12 +18,20 @@ using namespace mfem; using namespace mfem::future; using mfem::future::tensor; +#ifdef MFEM_USE_ENZYME +using dscalar_t = real_t; +#else +using mfem::future::dual; +using dscalar_t = dual; +#endif + namespace dfem_pa_kernels { template -void DFemMass(const char *filename, int p, const int r) +void dfem_mass(const char *filename, int p, const int r) { + constexpr int BDIM = DIM - 1; CAPTURE(filename, DIM, p, r); Mesh smesh(filename); @@ -33,16 +41,8 @@ void DFemMass(const char *filename, int p, const int r) p = std::max(p, pmesh.GetNodalFESpace()->GetMaxElementOrder()); smesh.Clear(); - Array all_domain_attr; - if (pmesh.attributes.Size() > 0) - { - all_domain_attr.SetSize(pmesh.attributes.Max()); - all_domain_attr = 1; - } - H1_FECollection fec(p, DIM); ParFiniteElementSpace fes(&pmesh, &fec); - const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p + r); ParGridFunction x(&fes), y(&fes), z(&fes); Vector X(fes.GetTrueVSize()), Y(fes.GetTrueVSize()), Z(fes.GetTrueVSize()); @@ -51,37 +51,114 @@ void DFemMass(const char *filename, int p, const int r) x.SetFromTrueDofs(X); ConstantCoefficient one(1.0); - ParBilinearForm blf(&fes); - blf.AddDomainIntegrator(new MassIntegrator(one, ir)); - blf.SetAssemblyLevel(AssemblyLevel::PARTIAL); - blf.Assemble(); - blf.Mult(x, y); - fes.GetProlongationMatrix()->MultTranspose(y, Y); - static constexpr int U = 0, Coords = 1; - const auto sol = std::vector{ FieldDescriptor{ U, &fes } }; - DifferentiableOperator dop(sol, {{Coords, nodes->ParFESpace()}}, pmesh); - const auto mf_mass_qf = - [] MFEM_HOST_DEVICE(const real_t &dudxi, - const tensor &J, const real_t &w) - { return tuple{dudxi * w * det(J)}; }; - dop.AddDomainIntegrator(mf_mass_qf, - tuple{ Value{}, Gradient{}, Weight{} }, - tuple{ Value{} }, - *ir, all_domain_attr); - dop.SetParameters({ nodes }); + // Test domain - fes.GetRestrictionMatrix()->Mult(x, X); - dop.Mult(X, Z); - Y -= Z; + SECTION("domain") + { + const auto *ir = &IntRules.Get(pmesh.GetTypicalElementGeometry(), 2 * p + r); - real_t norm_g, norm_l = Y.Normlinf(); - MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm()); - REQUIRE(norm_g == MFEM_Approx(0.0)); - MPI_Barrier(MPI_COMM_WORLD); + Array all_domain_attr; + if (pmesh.attributes.Size() > 0) + { + all_domain_attr.SetSize(pmesh.attributes.Max()); + all_domain_attr = 1; + } + + ParBilinearForm blf(&fes); + blf.AddDomainIntegrator(new MassIntegrator(one, ir)); + blf.SetAssemblyLevel(AssemblyLevel::PARTIAL); + blf.Assemble(); + blf.Mult(x, y); + fes.GetProlongationMatrix()->MultTranspose(y, Y); + + static constexpr int U = 0, Coords = 1; + const auto sol = std::vector{ FieldDescriptor{ U, &fes } }; + DifferentiableOperator dop(sol, {{Coords, nodes->ParFESpace()}}, pmesh); + const auto mf_mass_qf = + [] MFEM_HOST_DEVICE(const real_t &u, + const tensor &J, const real_t &w) + { return tuple{u * w * det(J)}; }; + dop.AddDomainIntegrator(mf_mass_qf, + tuple{ Value{}, Gradient{}, Weight{} }, + tuple{ Value{} }, + *ir, all_domain_attr); + dop.SetParameters({ nodes }); + + fes.GetRestrictionMatrix()->Mult(x, X); + dop.Mult(X, Z); + Y -= Z; + + real_t norm_g, norm_l = Y.Normlinf(); + MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm()); + REQUIRE(norm_g == MFEM_Approx(0.0)); + MPI_Barrier(MPI_COMM_WORLD); + } + + // Test boundary + // This ensures that we're not trying to test on fully periodic meshes + if (!((std::string("../../data/periodic-square.mesh").compare(filename) == 0) || + (std::string("../../data/periodic-cube.mesh").compare(filename) == 0))) + { + SECTION("boundary") + { + const auto *ir = &IntRules.Get(pmesh.GetTypicalFaceGeometry(), 2 * p + r); + + Array all_bdr_attr; + if (pmesh.bdr_attributes.Size() > 0) + { + all_bdr_attr.SetSize(pmesh.bdr_attributes.Max()); + all_bdr_attr = 1; + } + + ParBilinearForm blf(&fes); + blf.AddBoundaryIntegrator(new MassIntegrator(one, ir)); + blf.SetAssemblyLevel(AssemblyLevel::PARTIAL); + blf.Assemble(); + blf.Mult(x, y); + fes.GetProlongationMatrix()->MultTranspose(y, Y); + + static constexpr int U = 0, Coords = 1; + const auto sol = std::vector{FieldDescriptor{U, &fes}}; + DifferentiableOperator dop(sol, {{Coords, nodes->ParFESpace()}}, pmesh); + + const auto mf_mass_qf = + [] MFEM_HOST_DEVICE(const dscalar_t &u, + const tensor &J, + const real_t &w) + { + return tuple{u * weight(J) * w}; + }; + + auto derivatives = std::integer_sequence {}; + dop.AddBoundaryIntegrator(mf_mass_qf, + tuple{ Value{}, Gradient{}, Weight{} }, + tuple{ Value{} }, + *ir, all_bdr_attr, derivatives); + dop.SetParameters({nodes}); + + fes.GetRestrictionMatrix()->Mult(x, X); + dop.Mult(X, Z); + + Y -= Z; + real_t norm_g, norm_l = Y.Normlinf(); + MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm()); + REQUIRE(norm_g == MFEM_Approx(0.0)); + + auto dRdU = dop.GetDerivative(U, {&x}, {nodes}); + dRdU->Mult(X, Z); + + fes.GetProlongationMatrix()->MultTranspose(y, Y); + Y -= Z; + norm_l = Y.Normlinf(); + MPI_Allreduce(&norm_l, &norm_g, 1, MPI_DOUBLE, MPI_MAX, pmesh.GetComm()); + REQUIRE(norm_g == MFEM_Approx(0.0)); + MPI_Barrier(MPI_COMM_WORLD); + } + } } -TEST_CASE("DFEM Mass", "[Parallel][DFEM]") +TEST_CASE("dFEM Mass", "[Parallel][dFEM]") { const bool all_tests = launch_all_non_regression_tests; @@ -96,7 +173,7 @@ TEST_CASE("DFEM Mass", "[Parallel][DFEM]") "../../data/rt-2d-q3.mesh", "../../data/inline-quad.mesh", "../../data/periodic-square.mesh"); - DFemMass<2>(filename, p, r); + dfem_mass<2>(filename, p, r); } SECTION("3D p=" + std::to_string(p) + " r=" + std::to_string(r)) @@ -107,7 +184,7 @@ TEST_CASE("DFEM Mass", "[Parallel][DFEM]") "../../data/inline-hex.mesh", "../../data/toroid-hex.mesh", "../../data/periodic-cube.mesh"); - DFemMass<3>(filename, p, r); + dfem_mass<3>(filename, p, r); } } diff --git a/tests/unit/fem/make_permuted_mesh.cpp b/tests/unit/fem/make_permuted_mesh.cpp new file mode 100644 index 0000000000..00fbfe71f3 --- /dev/null +++ b/tests/unit/fem/make_permuted_mesh.cpp @@ -0,0 +1,162 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#include "make_permuted_mesh.hpp" + +namespace mfem +{ + +Mesh Mesh2D_Orientation(int face_perm_1, int face_perm_2) +{ + static const int dim = 2; + static const int nv = 6; + static const int nel = 2; + Mesh mesh(dim, nv, nel); + real_t x[dim]; + x[0] = 0.0; x[1] = 0.0; + mesh.AddVertex(x); + x[0] = 1.0; x[1] = 0.0; + mesh.AddVertex(x); + x[0] = 2.0; x[1] = 0.0; + mesh.AddVertex(x); + x[0] = 0.0; x[1] = 1.0; + mesh.AddVertex(x); + x[0] = 1.0; x[1] = 1.0; + mesh.AddVertex(x); + x[0] = 2.0; x[1] = 1.0; + mesh.AddVertex(x); + int el[4]; + el[0] = 0; + el[1] = 1; + el[2] = 4; + el[3] = 3; + std::rotate(&el[0], &el[face_perm_1], &el[3] + 1); + + mesh.AddQuad(el); + + el[0] = 1; + el[1] = 2; + el[2] = 5; + el[3] = 4; + std::rotate(&el[0], &el[face_perm_2], &el[3] + 1); + mesh.AddQuad(el); + + mesh.FinalizeQuadMesh(true); + mesh.GenerateBoundaryElements(); + mesh.Finalize(); + return mesh; +} + +void Rotation3DVertices(int *v, int ref_face, int rot) +{ + std::vector face_1, face_2; + + switch (ref_face/2) + { + case 0: + face_1 = {v[0], v[1], v[2], v[3]}; + face_2 = {v[4], v[5], v[6], v[7]}; + break; + case 1: + face_1 = {v[1], v[5], v[6], v[2]}; + face_2 = {v[0], v[4], v[7], v[3]}; + break; + case 2: + face_1 = {v[4], v[5], v[1], v[0]}; + face_2 = {v[7], v[6], v[2], v[3]}; + break; + } + if (ref_face % 2 == 0) + { + std::reverse(face_1.begin(), face_1.end()); + std::reverse(face_2.begin(), face_2.end()); + std::swap(face_1, face_2); + } + + std::rotate(face_1.begin(), face_1.begin() + rot, face_1.end()); + std::rotate(face_2.begin(), face_2.begin() + rot, face_2.end()); + + for (int i=0; i<4; ++i) + { + v[i] = face_1[i]; + v[i+4] = face_2[i]; + } +} + +Mesh Mesh3D_Orientation(int face_perm_1, int face_perm_2) +{ + static const int dim = 3; + static const int nv = 12; + static const int nel = 2; + Mesh mesh(dim, nv, nel); + real_t x[dim]; + x[0] = 0.0; x[1] = 0.0; x[2] = 0.0; + mesh.AddVertex(x); + x[0] = 1.0; x[1] = 0.0; x[2] = 0.0; + mesh.AddVertex(x); + x[0] = 2.0; x[1] = 0.0; x[2] = 0.0; + mesh.AddVertex(x); + x[0] = 0.0; x[1] = 1.0; x[2] = 0.0; + mesh.AddVertex(x); + x[0] = 1.0; x[1] = 1.0; x[2] = 0.0; + mesh.AddVertex(x); + x[0] = 2.0; x[1] = 1.0; x[2] = 0.0; + mesh.AddVertex(x); + x[0] = 0.0; x[1] = 0.0; x[2] = 1.0; + mesh.AddVertex(x); + x[0] = 1.0; x[1] = 0.0; x[2] = 1.0; + mesh.AddVertex(x); + x[0] = 2.0; x[1] = 0.0; x[2] = 1.0; + mesh.AddVertex(x); + x[0] = 0.0; x[1] = 1.0; x[2] = 1.0; + mesh.AddVertex(x); + x[0] = 1.0; x[1] = 1.0; x[2] = 1.0; + mesh.AddVertex(x); + x[0] = 2.0; x[1] = 1.0; x[2] = 1.0; + mesh.AddVertex(x); + + int el[8]; + + el[0] = 0; + el[1] = 1; + el[2] = 4; + el[3] = 3; + el[4] = 6; + el[5] = 7; + el[6] = 10; + el[7] = 9; + Rotation3DVertices(el, face_perm_1/4, face_perm_1%4); + mesh.AddHex(el); + + el[0] = 1; + el[1] = 2; + el[2] = 5; + el[3] = 4; + el[4] = 7; + el[5] = 8; + el[6] = 11; + el[7] = 10; + Rotation3DVertices(el, face_perm_2/4, face_perm_2%4); + mesh.AddHex(el); + + mesh.FinalizeHexMesh(true); + mesh.Finalize(); + return mesh; +} + +Mesh MeshOrientation(int dim, int o1, int o2) +{ + if (dim == 2) { return Mesh2D_Orientation(o1, o2); } + else if (dim == 3) { return Mesh3D_Orientation(o1, o2); } + else { MFEM_ABORT("Unsupported dimension."); } +} + +} diff --git a/tests/unit/fem/make_permuted_mesh.hpp b/tests/unit/fem/make_permuted_mesh.hpp new file mode 100644 index 0000000000..1f7836332c --- /dev/null +++ b/tests/unit/fem/make_permuted_mesh.hpp @@ -0,0 +1,25 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_MAKE_PERMUTED_MESH_HPP +#define MFEM_MAKE_PERMUTED_MESH_HPP + +#include "mfem.hpp" +#include "unit_tests.hpp" + +namespace mfem +{ + +Mesh MeshOrientation(int dim, int o1, int o2); + +} + +#endif diff --git a/tests/unit/fem/test_cmplx_gridfunc_save.cpp b/tests/unit/fem/test_cmplx_gridfunc_save.cpp new file mode 100644 index 0000000000..a21d7c2363 --- /dev/null +++ b/tests/unit/fem/test_cmplx_gridfunc_save.cpp @@ -0,0 +1,202 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#include "mfem.hpp" +#include "unit_tests.hpp" + +using namespace mfem; + +TEST_CASE("ComplexGridFunction Save", "[ComplexGridFunction]") +{ + const int order = 3; + + Mesh mesh = Mesh::MakeCartesian3D(1, 2, 3, Element::PYRAMID, + 1.0, 2.0, 3.0); + + H1_FECollection fec_h1(order, mesh.Dimension()); + ND_FECollection fec_nd(order, mesh.Dimension()); + RT_FECollection fec_rt(order-1, mesh.Dimension()); + L2_FECollection fec_l2(order-1, mesh.Dimension()); + + FiniteElementSpace fes_h1(&mesh, &fec_h1); + FiniteElementSpace fes_nd(&mesh, &fec_nd); + FiniteElementSpace fes_rt(&mesh, &fec_rt); + FiniteElementSpace fes_l2(&mesh, &fec_l2); + + ComplexGridFunction gf_h1(&fes_h1); + ComplexGridFunction gf_nd(&fes_nd); + ComplexGridFunction gf_rt(&fes_rt); + ComplexGridFunction gf_l2(&fes_l2); + + gf_h1.Randomize(1); + gf_nd.Randomize(1); + gf_rt.Randomize(1); + gf_l2.Randomize(1); + + Vector zeroVec(3); zeroVec = 0.0; + ConstantCoefficient zeroCoef(0.0); + VectorConstantCoefficient zeroVecCoef(zeroVec); + + const double norm_h1 = gf_h1.ComputeL2Error(zeroCoef, zeroCoef); + const double norm_nd = gf_nd.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double norm_rt = gf_rt.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double norm_l2 = gf_l2.ComputeL2Error(zeroCoef, zeroCoef); + + std::ofstream ofs_h1("cmplx_h1.gf"); ofs_h1.precision(8); + std::ofstream ofs_nd("cmplx_nd.gf"); ofs_nd.precision(8); + std::ofstream ofs_rt("cmplx_rt.gf"); ofs_rt.precision(8); + std::ofstream ofs_l2("cmplx_l2.gf"); ofs_l2.precision(8); + + gf_h1.Save(ofs_h1); ofs_h1.close(); + gf_nd.Save(ofs_nd); ofs_nd.close(); + gf_rt.Save(ofs_rt); ofs_rt.close(); + gf_l2.Save(ofs_l2); ofs_l2.close(); + + std::ifstream ifs_h1("cmplx_h1.gf"); + std::ifstream ifs_nd("cmplx_nd.gf"); + std::ifstream ifs_rt("cmplx_rt.gf"); + std::ifstream ifs_l2("cmplx_l2.gf"); + + ComplexGridFunction gf_h1_read(&mesh, ifs_h1); ifs_h1.close(); + ComplexGridFunction gf_nd_read(&mesh, ifs_nd); ifs_nd.close(); + ComplexGridFunction gf_rt_read(&mesh, ifs_rt); ifs_rt.close(); + ComplexGridFunction gf_l2_read(&mesh, ifs_l2); ifs_l2.close(); + + gf_h1_read -= gf_h1; + gf_nd_read -= gf_nd; + gf_rt_read -= gf_rt; + gf_l2_read -= gf_l2; + + const double diff_h1 = gf_h1_read.ComputeL2Error(zeroCoef, zeroCoef); + const double diff_nd = gf_nd_read.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double diff_rt = gf_rt_read.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double diff_l2 = gf_l2_read.ComputeL2Error(zeroCoef, zeroCoef); + + REQUIRE(diff_h1 < 1e-8 * norm_h1); + REQUIRE(diff_nd < 1e-8 * norm_nd); + REQUIRE(diff_rt < 1e-8 * norm_rt); + REQUIRE(diff_l2 < 1e-8 * norm_l2); + + // Clean up + REQUIRE(std::remove("cmplx_h1.gf") == 0); + REQUIRE(std::remove("cmplx_nd.gf") == 0); + REQUIRE(std::remove("cmplx_rt.gf") == 0); + REQUIRE(std::remove("cmplx_l2.gf") == 0); +} + +#ifdef MFEM_USE_MPI + +TEST_CASE("ParComplexGridFunction Save", "[ParComplexGridFunction][Parallel]") +{ + const int num_procs = Mpi::WorldSize(); + const int my_rank = Mpi::WorldRank(); + const int order = 3; + + const int nx = (int)rint(cbrt(real_t(num_procs))); + const int ny = (int)rint(2.0 * cbrt(real_t(num_procs))); + const int nz = (int)rint(3.0 * cbrt(real_t(num_procs))); + Mesh mesh = Mesh::MakeCartesian3D(nx, ny, nz, Element::PYRAMID, + 1.0, 2.0, 3.0); + + // Define a parallel mesh by a partitioning of the serial mesh. + ParMesh pmesh(MPI_COMM_WORLD, mesh); + + H1_FECollection fec_h1(order, mesh.Dimension()); + ND_FECollection fec_nd(order, mesh.Dimension()); + RT_FECollection fec_rt(order-1, mesh.Dimension()); + L2_FECollection fec_l2(order-1, mesh.Dimension()); + + ParFiniteElementSpace pfes_h1(&pmesh, &fec_h1); + ParFiniteElementSpace pfes_nd(&pmesh, &fec_nd); + ParFiniteElementSpace pfes_rt(&pmesh, &fec_rt); + ParFiniteElementSpace pfes_l2(&pmesh, &fec_l2); + + ParComplexGridFunction pgf_h1(&pfes_h1); + ParComplexGridFunction pgf_nd(&pfes_nd); + ParComplexGridFunction pgf_rt(&pfes_rt); + ParComplexGridFunction pgf_l2(&pfes_l2); + + pgf_h1.Randomize(1); + pgf_nd.Randomize(1); + pgf_rt.Randomize(1); + pgf_l2.Randomize(1); + + // Ensure that the L-DOFs are set consistently on all ranks + pgf_h1.real().SetTrueVector(); pgf_h1.real().SetFromTrueVector(); + pgf_h1.imag().SetTrueVector(); pgf_h1.imag().SetFromTrueVector(); + pgf_nd.real().SetTrueVector(); pgf_nd.real().SetFromTrueVector(); + pgf_nd.imag().SetTrueVector(); pgf_nd.imag().SetFromTrueVector(); + pgf_rt.real().SetTrueVector(); pgf_rt.real().SetFromTrueVector(); + pgf_rt.imag().SetTrueVector(); pgf_rt.imag().SetFromTrueVector(); + pgf_l2.real().SetTrueVector(); pgf_l2.real().SetFromTrueVector(); + pgf_l2.imag().SetTrueVector(); pgf_l2.imag().SetFromTrueVector(); + + Vector zeroVec(3); zeroVec = 0.0; + ConstantCoefficient zeroCoef(0.0); + VectorConstantCoefficient zeroVecCoef(zeroVec); + + const double norm_h1 = pgf_h1.ComputeL2Error(zeroCoef, zeroCoef); + const double norm_nd = pgf_nd.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double norm_rt = pgf_rt.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double norm_l2 = pgf_l2.ComputeL2Error(zeroCoef, zeroCoef); + + std::ostringstream name_h1, name_nd, name_rt, name_l2; + name_h1 << "cmplx_gf_h1." << std::setfill('0') << std::setw(6) << my_rank; + name_nd << "cmplx_gf_nd." << std::setfill('0') << std::setw(6) << my_rank; + name_rt << "cmplx_gf_rt." << std::setfill('0') << std::setw(6) << my_rank; + name_l2 << "cmplx_gf_l2." << std::setfill('0') << std::setw(6) << my_rank; + + std::ofstream ofs_h1(name_h1.str().c_str()); ofs_h1.precision(8); + std::ofstream ofs_nd(name_nd.str().c_str()); ofs_nd.precision(8); + std::ofstream ofs_rt(name_rt.str().c_str()); ofs_rt.precision(8); + std::ofstream ofs_l2(name_l2.str().c_str()); ofs_l2.precision(8); + + pgf_h1.Save(ofs_h1); ofs_h1.close(); + pgf_nd.Save(ofs_nd); ofs_nd.close(); + pgf_rt.Save(ofs_rt); ofs_rt.close(); + pgf_l2.Save(ofs_l2); ofs_l2.close(); + + std::ifstream ifs_h1(name_h1.str().c_str()); + std::ifstream ifs_nd(name_nd.str().c_str()); + std::ifstream ifs_rt(name_rt.str().c_str()); + std::ifstream ifs_l2(name_l2.str().c_str()); + + ParComplexGridFunction pgf_h1_read(&pmesh, ifs_h1); ifs_h1.close(); + ParComplexGridFunction pgf_nd_read(&pmesh, ifs_nd); ifs_nd.close(); + ParComplexGridFunction pgf_rt_read(&pmesh, ifs_rt); ifs_rt.close(); + ParComplexGridFunction pgf_l2_read(&pmesh, ifs_l2); ifs_l2.close(); + + pgf_h1_read -= pgf_h1; + pgf_nd_read -= pgf_nd; + pgf_rt_read -= pgf_rt; + pgf_l2_read -= pgf_l2; + + const double diff_h1 = pgf_h1_read.ComputeL2Error(zeroCoef, zeroCoef); + const double diff_nd = pgf_nd_read.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double diff_rt = pgf_rt_read.ComputeL2Error(zeroVecCoef, zeroVecCoef); + const double diff_l2 = pgf_l2_read.ComputeL2Error(zeroCoef, zeroCoef); + + if (my_rank == 0) + { + REQUIRE(diff_h1 < 1e-8 * norm_h1); + REQUIRE(diff_nd < 1e-8 * norm_nd); + REQUIRE(diff_rt < 1e-8 * norm_rt); + REQUIRE(diff_l2 < 1e-8 * norm_l2); + } + + // Clean up + REQUIRE(std::remove(name_h1.str().c_str()) == 0); + REQUIRE(std::remove(name_nd.str().c_str()) == 0); + REQUIRE(std::remove(name_rt.str().c_str()) == 0); + REQUIRE(std::remove(name_l2.str().c_str()) == 0); +} + +#endif // MFEM_USE_MPI diff --git a/tests/unit/fem/test_coefficient.cpp b/tests/unit/fem/test_coefficient.cpp index 07e96cacf3..7037b61c55 100644 --- a/tests/unit/fem/test_coefficient.cpp +++ b/tests/unit/fem/test_coefficient.cpp @@ -391,3 +391,110 @@ TEST_CASE("Symmetric Matrix Coefficient", "[Coefficient]") // Require equality REQUIRE(qf.DistanceTo(values) == MFEM_Approx(0.0)); } + +TEST_CASE("Piecewise Constant Coefficient", "[Coefficient]") +{ + Mesh mesh("../../data/beam-quad.mesh"); + + QuadratureSpace qs(&mesh, 2); + FaceQuadratureSpace qs_f(mesh, 2, FaceType::Boundary); + QuadratureFunction qf(qs); + QuadratureFunction qf_f(qs_f); + + Vector values({1.0, 2.0, 3.0}); + PWConstCoefficient coeff(values); + + coeff.Project(qf); + for (int e = 0; e < mesh.GetNE(); ++e) + { + Vector vals; + qf.GetValues(e, vals); + const int a = mesh.GetAttribute(e); + for (const real_t val : vals) + { + REQUIRE(val == a); + } + } + + coeff.Project(qf_f); + for (int be = 0; be < mesh.GetNBE(); ++be) + { + const int f = mesh.GetBdrElementFaceIndex(be); + const int bf = mesh.GetInvFaceIndices(FaceType::Boundary).at(f); + Vector vals; + qf_f.GetValues(bf, vals); + const int a = mesh.GetBdrAttribute(be); + for (const real_t val : vals) + { + REQUIRE(val == a); + } + } +} + +TEST_CASE("Project Sum/Product/Ratio Coefficients", "[Coefficient][GPU]") +{ + // Small mesh with a few elements + Mesh mesh = Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL); + + // Use low-order quadrature space so qf has a few points + QuadratureSpace qs(&mesh, 2); + + QuadratureFunction qf1(qs); + qf1.Randomize(); + QuadratureFunctionCoefficient qf_coeff_1(qf1); + + QuadratureFunction qf2(qs); + qf2.Randomize(); + QuadratureFunctionCoefficient qf_coeff_2(qf2); + + auto check_coeff = [&](Coefficient &coeff) + { + QuadratureFunction qf(qs); + coeff.Project(qf); + qf.HostRead(); + Vector vals; + for (int e = 0; e < qs.GetNE(); ++e) + { + const IntegrationRule &ir = qs.GetIntRule(e); + ElementTransformation &T = *qs.GetTransformation(e); + qf.GetValues(e, vals); + for (int iq = 0; iq < ir.Size(); ++iq) + { + const real_t val = coeff.Eval(T, ir[iq]); + REQUIRE(val == MFEM_Approx(AsConst(vals)[iq])); + } + } + }; + + SECTION("SumCoefficient") + { + SumCoefficient s1(2.2, qf_coeff_2, 3.3, 4.4); + SumCoefficient s2(qf_coeff_1, qf_coeff_2, 3.3, 4.4); + + check_coeff(s1); + check_coeff(s2); + } + + SECTION("ProductCoefficient") + { + ProductCoefficient p1(2.2, qf_coeff_2); + ProductCoefficient p2(qf_coeff_1, qf_coeff_2); + + check_coeff(p1); + check_coeff(p2); + } + + SECTION("RatioCoefficient") + { + RatioCoefficient r1(1.1, qf_coeff_2); + RatioCoefficient r2(qf_coeff_1, 2.2); + RatioCoefficient r3(qf_coeff_1, qf_coeff_2); + + check_coeff(r1); + check_coeff(r2); + check_coeff(r3); + + r1.SetBConst(2.2); + check_coeff(r1); + } +} diff --git a/tests/unit/fem/test_col_lag_der.cpp b/tests/unit/fem/test_col_lag_der.cpp index 280bfb6250..3680ad56b2 100644 --- a/tests/unit/fem/test_col_lag_der.cpp +++ b/tests/unit/fem/test_col_lag_der.cpp @@ -9,8 +9,8 @@ // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. -#include "unit_tests.hpp" #include "mfem.hpp" +#include "unit_tests.hpp" #include "fem/qinterp/grad.hpp" diff --git a/tests/unit/fem/test_face_permutation.cpp b/tests/unit/fem/test_face_permutation.cpp index 82e4c7af8e..714d0faff0 100644 --- a/tests/unit/fem/test_face_permutation.cpp +++ b/tests/unit/fem/test_face_permutation.cpp @@ -11,148 +11,10 @@ #include "mfem.hpp" #include "unit_tests.hpp" +#include "make_permuted_mesh.hpp" using namespace mfem; -Mesh *mesh_2d_orientation(int face_perm_1, int face_perm_2) -{ - static const int dim = 2; - static const int nv = 6; - static const int nel = 2; - Mesh *mesh = new Mesh(dim, nv, nel); - real_t x[dim]; - x[0] = 0.0; x[1] = 0.0; - mesh->AddVertex(x); - x[0] = 1.0; x[1] = 0.0; - mesh->AddVertex(x); - x[0] = 2.0; x[1] = 0.0; - mesh->AddVertex(x); - x[0] = 0.0; x[1] = 1.0; - mesh->AddVertex(x); - x[0] = 1.0; x[1] = 1.0; - mesh->AddVertex(x); - x[0] = 2.0; x[1] = 1.0; - mesh->AddVertex(x); - int el[4]; - el[0] = 0; - el[1] = 1; - el[2] = 4; - el[3] = 3; - std::rotate(&el[0], &el[face_perm_1], &el[3] + 1); - - mesh->AddQuad(el); - - el[0] = 1; - el[1] = 2; - el[2] = 5; - el[3] = 4; - std::rotate(&el[0], &el[face_perm_2], &el[3] + 1); - mesh->AddQuad(el); - - mesh->FinalizeQuadMesh(true); - mesh->GenerateBoundaryElements(); - mesh->Finalize(); - return mesh; -} - -void rotate_3d_vertices(int *v, int ref_face, int rot) -{ - std::vector face_1, face_2; - - switch (ref_face/2) - { - case 0: - face_1 = {v[0], v[1], v[2], v[3]}; - face_2 = {v[4], v[5], v[6], v[7]}; - break; - case 1: - face_1 = {v[1], v[5], v[6], v[2]}; - face_2 = {v[0], v[4], v[7], v[3]}; - break; - case 2: - face_1 = {v[4], v[5], v[1], v[0]}; - face_2 = {v[7], v[6], v[2], v[3]}; - break; - } - if (ref_face % 2 == 0) - { - std::reverse(face_1.begin(), face_1.end()); - std::reverse(face_2.begin(), face_2.end()); - std::swap(face_1, face_2); - } - - std::rotate(face_1.begin(), face_1.begin() + rot, face_1.end()); - std::rotate(face_2.begin(), face_2.begin() + rot, face_2.end()); - - for (int i=0; i<4; ++i) - { - v[i] = face_1[i]; - v[i+4] = face_2[i]; - } -} - -Mesh *mesh_3d_orientation(int face_perm_1, int face_perm_2) -{ - static const int dim = 3; - static const int nv = 12; - static const int nel = 2; - Mesh *mesh = new Mesh(dim, nv, nel); - real_t x[dim]; - x[0] = 0.0; x[1] = 0.0; x[2] = 0.0; - mesh->AddVertex(x); - x[0] = 1.0; x[1] = 0.0; x[2] = 0.0; - mesh->AddVertex(x); - x[0] = 2.0; x[1] = 0.0; x[2] = 0.0; - mesh->AddVertex(x); - x[0] = 0.0; x[1] = 1.0; x[2] = 0.0; - mesh->AddVertex(x); - x[0] = 1.0; x[1] = 1.0; x[2] = 0.0; - mesh->AddVertex(x); - x[0] = 2.0; x[1] = 1.0; x[2] = 0.0; - mesh->AddVertex(x); - x[0] = 0.0; x[1] = 0.0; x[2] = 1.0; - mesh->AddVertex(x); - x[0] = 1.0; x[1] = 0.0; x[2] = 1.0; - mesh->AddVertex(x); - x[0] = 2.0; x[1] = 0.0; x[2] = 1.0; - mesh->AddVertex(x); - x[0] = 0.0; x[1] = 1.0; x[2] = 1.0; - mesh->AddVertex(x); - x[0] = 1.0; x[1] = 1.0; x[2] = 1.0; - mesh->AddVertex(x); - x[0] = 3.0; x[1] = 1.0; x[2] = 1.0; - mesh->AddVertex(x); - - int el[8]; - - el[0] = 0; - el[1] = 1; - el[2] = 4; - el[3] = 3; - el[4] = 6; - el[5] = 7; - el[6] = 10; - el[7] = 9; - rotate_3d_vertices(el, face_perm_1/4, face_perm_1%4); - mesh->AddHex(el); - - el[0] = 1; - el[1] = 2; - el[2] = 5; - el[3] = 4; - el[4] = 7; - el[5] = 8; - el[6] = 11; - el[7] = 10; - rotate_3d_vertices(el, face_perm_2/4, face_perm_2%4); - mesh->AddHex(el); - - mesh->FinalizeHexMesh(true); - mesh->GenerateBoundaryElements(); - mesh->Finalize(); - return mesh; -} - real_t x_fn(const Vector &xvec) { return xvec[0]; } real_t y_fn(const Vector &xvec) { return xvec[1]; } real_t z_fn(const Vector &xvec) { return xvec[2]; } @@ -214,10 +76,9 @@ TEST_CASE("2D Face Permutation", "[Face Permutation]") { for (int fp1=0; fp1<4; ++fp1) { - Mesh *mesh = mesh_2d_orientation(fp1, fp2); - real_t error = TestFaceRestriction(*mesh, order); + Mesh mesh = MeshOrientation(2, fp1, fp2); + real_t error = TestFaceRestriction(mesh, order); max_err = std::max(max_err, error); - delete mesh; } } REQUIRE(max_err < 1e-15); @@ -231,10 +92,9 @@ TEST_CASE("3D Face Permutation", "[Face Permutation]") { for (int fp1=0; fp1<24; ++fp1) { - Mesh *mesh = mesh_3d_orientation(fp1, fp2); - real_t error = TestFaceRestriction(*mesh, order); + Mesh mesh = MeshOrientation(3, fp1, fp2); + real_t error = TestFaceRestriction(mesh, order); max_err = std::max(max_err, error); - delete mesh; } } REQUIRE(max_err < 1e-15); diff --git a/tests/unit/fem/test_get_value.cpp b/tests/unit/fem/test_get_value.cpp index 76b5f15e1e..a4a19f5a69 100644 --- a/tests/unit/fem/test_get_value.cpp +++ b/tests/unit/fem/test_get_value.cpp @@ -398,7 +398,7 @@ TEST_CASE("1D GetValue", } #ifdef MFEM_USE_MPI -# + TEST_CASE("1D GetValue in Parallel", "[ParGridFunction]" "[GridFunctionCoefficient]" @@ -869,7 +869,7 @@ TEST_CASE("2D GetValue", } #ifdef MFEM_USE_MPI -# + TEST_CASE("2D GetValue in Parallel", "[ParGridFunction]" "[GridFunctionCoefficient]" @@ -1375,7 +1375,7 @@ TEST_CASE("3D GetValue", } #ifdef MFEM_USE_MPI -# + TEST_CASE("3D GetValue in Parallel", "[ParGridFunction]" "[GridFunctionCoefficient]" @@ -2074,7 +2074,7 @@ TEST_CASE("2D GetVectorValue", } #ifdef MFEM_USE_MPI -# + TEST_CASE("2D GetVectorValue in Parallel", "[ParGridFunction]" "[VectorGridFunctionCoefficient]" @@ -3033,7 +3033,7 @@ TEST_CASE("3D GetVectorValue", } #ifdef MFEM_USE_MPI -# + TEST_CASE("3D GetVectorValue in Parallel", "[ParGridFunction]" "[VectorGridFunctionCoefficient]" diff --git a/tests/unit/fem/test_getgradients.cpp b/tests/unit/fem/test_getgradients.cpp new file mode 100644 index 0000000000..eddf7f1f33 --- /dev/null +++ b/tests/unit/fem/test_getgradients.cpp @@ -0,0 +1,126 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#include "mfem.hpp" +#include "unit_tests.hpp" +#include + +using namespace mfem; + +Mesh MakePerturbedMesh(Geometry::Type geom, int mesh_p) +{ + MFEM_VERIFY(Geometry::POINT < geom && geom < Geometry::NUM_GEOMETRIES, + "invalid geom: " << geom); + const int dim = Geometry::Dimension[geom]; + const Element::Type type = Element::TypeFromGeometry(geom); + Mesh mesh = (dim == 1) ? Mesh::MakeCartesian1D(7) : + (dim == 2) ? Mesh::MakeCartesian2D(3, 5, type, true) : + Mesh::MakeCartesian3D(2, 3, 3, type); + mesh.SetCurvature(mesh_p); // mesh_p <= 0 means no nodes + + real_t h_min = infinity(); + for (int i = 0; i < mesh.GetNE(); i++) + { + h_min = std::fmin(h_min, mesh.GetElementSize(i, 1)); + } + if (mesh_p > 0) + { + h_min /= mesh_p; + } + + Vector pert(mesh.GetNodes() ? mesh.GetNodes()->Size() : mesh.GetNV()*dim); + pert.Randomize(473'099'612); + pert -= 0.5_r; + pert *= 0.25_r*h_min; + + mesh.MoveNodes(pert); + + return mesh; +} + +real_t CompareGradients(const GridFunction &f, + const IntegrationRule &ir, + const Vector &grad_f, + QVectorLayout ql = QVectorLayout::byNODES) +{ + real_t max_rel_error = 0; + + const FiniteElementSpace &fes = *f.FESpace(); + const int dim = fes.GetMesh()->Dimension(); + const int NE = fes.GetNE(); + const int NQ = ir.GetNPoints(); + + MFEM_VERIFY(fes.GetVDim() == 1, "only vdim == 1 is implemented!"); + + Vector grad_error(dim), grad; + DenseMatrix loc_grad; + grad_f.HostRead(); + for (int i = 0; i < NE; i++) + { + f.GetGradients(i, ir, loc_grad); // loc_grad is (dim x NQ) + + // The layout of grad_f is: + // ql == QVectorLayout::byNODES : NQ x VDIM x DIM x NE + // ql == QVectorLayout::byVDIM : VDIM x DIM x NQ x NE + + real_t max_error = 0, max_grad_norm = 0; + for (int j = 0; j < NQ; j++) + { + loc_grad.GetColumnReference(j, grad); + for (int d = 0; d < dim; d++) + { + // vdim is 1 + real_t g = (ql == QVectorLayout::byNODES) ? + grad_f[j + NQ*(d + dim*i)] : + grad_f[d + dim*(j + NQ*i)]; + grad_error[d] = g - grad(d); + } + max_error = std::fmax(max_error, grad_error.Norml2()); + max_grad_norm = std::fmax(max_grad_norm, grad.Norml2()); + } + real_t rel_error = // element relative error + (max_grad_norm > 0_r) ? max_error/max_grad_norm : + (max_error == 0_r) ? 0_r : infinity(); + max_rel_error = std::fmax(max_rel_error, rel_error); + } + + return max_rel_error; +} + +real_t RandReal() +{ + static std::mt19937_64 gen(8'656'127'438'685'088'196); + static std::uniform_real_distribution dis_real(0_r, 1_r); // [0, 1) + return dis_real(gen); +} + +TEST_CASE("GetGradients All Elements", "[GridFunction][GPU]") +{ + auto geom = GENERATE(range(int(Geometry::SEGMENT), + int(Geometry::NUM_GEOMETRIES))); + auto mesh_p = GENERATE(0, 2); + auto p = GENERATE(1, 3); + + Mesh mesh = MakePerturbedMesh((Geometry::Type)geom, mesh_p); + H1_FECollection h1_fec(p, Geometry::Dimension[geom]); + FiniteElementSpace h1_fes(&mesh, &h1_fec); + GridFunction h1_gf(&h1_fes); + for (auto &d : h1_gf) { d = RandReal(); } + Vector grad_h1_gf; + const IntegrationRule &ir = IntRules.Get(geom, 2*p+1); + for (auto ql : {QVectorLayout::byNODES, QVectorLayout::byVDIM}) + { + h1_gf.GetGradients(ir, grad_h1_gf, ql); + real_t rel_err = CompareGradients(h1_gf, ir, grad_h1_gf, ql); + CAPTURE(geom, mesh_p, p, ql); + CHECK(rel_err == MFEM_Approx(0_r)); + } +} diff --git a/tests/unit/fem/test_lor_batched.cpp b/tests/unit/fem/test_lor_batched.cpp index edab89d32e..bdcec76893 100644 --- a/tests/unit/fem/test_lor_batched.cpp +++ b/tests/unit/fem/test_lor_batched.cpp @@ -11,6 +11,7 @@ #include "mfem.hpp" #include "unit_tests.hpp" +#include "../linalg/test_same_matrices.hpp" #include "../../fem/lor/lor_ads.hpp" #include "../../fem/lor/lor_ams.hpp" #include @@ -25,47 +26,6 @@ using namespace mfem; namespace lor_batched { -void TestSameMatrices(SparseMatrix &A1, const SparseMatrix &A2, - HYPRE_BigInt *cmap1=nullptr, - std::unordered_map *cmap2inv=nullptr) -{ - REQUIRE(A1.Height() == A2.Height()); - int n = A1.Height(); - - const int *I1 = A1.HostReadI(); - const int *J1 = A1.HostReadJ(); - const real_t *V1 = A1.HostReadData(); - - A2.HostReadI(); - A2.HostReadJ(); - A2.HostReadData(); - - real_t error = 0.0; - - for (int i=0; icount(cmap1[j]) > 0) - { - j = (*cmap2inv)[cmap1[j]]; - } - else - { - error = std::max(error, std::fabs(V1[jj])); - continue; - } - } - error = std::max(error, std::fabs(V1[jj] - A2(i,j))); - } - } - - REQUIRE(error == MFEM_Approx(0.0, 1e-10)); -} - template FE_COLL *NewLOR_FE_Collection(int order, int dim) { @@ -157,30 +117,6 @@ TEST_CASE("LOR Batched RT", "[LOR][BatchedLOR][GPU]") #ifdef MFEM_USE_MPI -void TestSameMatrices(HypreParMatrix &A1, const HypreParMatrix &A2) -{ - HYPRE_BigInt *cmap1, *cmap2; - SparseMatrix diag1, offd1, diag2, offd2; - - A1.GetDiag(diag1); - A2.GetDiag(diag2); - A1.GetOffd(offd1, cmap1); - A2.GetOffd(offd2, cmap2); - - TestSameMatrices(diag1, diag2); - - if (cmap1) - { - std::unordered_map cmap2inv; - for (int i=0; i void ParTestBatchedLOR() { diff --git a/tests/unit/fem/test_lor_dg.cpp b/tests/unit/fem/test_lor_dg.cpp new file mode 100644 index 0000000000..68bcf38f76 --- /dev/null +++ b/tests/unit/fem/test_lor_dg.cpp @@ -0,0 +1,382 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#include "mfem.hpp" +#include "unit_tests.hpp" +#include "make_permuted_mesh.hpp" +#include "../linalg/test_same_matrices.hpp" + +using namespace mfem; + +class DG_LOR_DiffusionPreconditioner : public BilinearFormIntegrator +{ + Mesh &mesh; + double kappa; + int p; + IntegrationRule gl_p, gl_pp1; + Vector shape1, shape2, nor; + +public: + DG_LOR_DiffusionPreconditioner(Mesh &mesh_, int p_, double kappa_) + : mesh(mesh_), kappa(kappa_), p(p_) + { + QuadratureFunctions1D::GaussLobatto(p+1, &gl_p); + QuadratureFunctions1D::GaussLobatto(p+2, &gl_pp1); + } + + double PenaltyFactor(int idx1, int idx2) + { + int pp1 = p + 1; + + int x1 = idx1 % pp1; + int y1 = (idx1 / pp1) % pp1; + int z1 = (idx1 / pp1) / pp1; + + int x2 = idx2 % pp1; + int y2 = (idx2 / pp1) % pp1; + int z2 = (idx2 / pp1) / pp1; + + int dim = mesh.Dimension(); + + auto compute_factor = [&](int i1, int i2) + { + int j = std::min(i1, i2); + if (i1 == i2) + { + double w = gl_p[j].weight; + double k = gl_pp1[i1+1].x - gl_pp1[i1].x; + return w/k; + } + else + { + double h = gl_p[j+1].x - gl_p[j].x; + double k1 = gl_pp1[i1+1].x - gl_pp1[i1].x; + double k2 = gl_pp1[i2+1].x - gl_pp1[i2].x; + double avg = 0.5*k1 + 0.5*k2; + return avg/h; + } + }; + + double factor = compute_factor(x1, x2); + if (dim >= 2) { factor *= compute_factor(y1, y2); } + if (dim == 3) { factor *= compute_factor(z1, z2); } + + return factor; + } + + double BdrPenaltyFactor(int idx, int f) + { + int pp1 = p+1; + + int x = idx % pp1; + int y = (idx / pp1) % pp1; + int z = (idx / pp1) / pp1; + + int dim = mesh.Dimension(); + + auto subcell_size = [&](int i) + { + return gl_pp1[i+1].x - gl_pp1[i].x; + }; + + double factor = (p+1)*(p+1); + if (dim == 1) + { + factor *= subcell_size(x); + } + else if (dim == 2) + { + int ni, nj; + ni = (f == 1 || f == 3) ? x : y; + nj = (f == 1 || f == 3) ? y : x; + factor *= subcell_size(ni)/subcell_size(nj)*gl_p[nj].weight; + } + else if (dim == 3) + { + int ni, nj, nk; + if (f == 2 || f == 4) { ni = x; nj = y; nk = z; } + else if (f == 1 || f == 3) { ni = y; nj = x; nk = z; } + else { ni = z; nj = x; nk = y; } + factor *= subcell_size(ni)/subcell_size(nj)/subcell_size(nk); + factor *= gl_p[nj].weight*gl_p[nk].weight; + } + + return factor; + } + + using BilinearFormIntegrator::AssembleFaceMatrix; + virtual void AssembleFaceMatrix(const FiniteElement &el1, + const FiniteElement &el2, + FaceElementTransformations &Trans, + DenseMatrix &elmat) override + { + int dim, ndof1, ndof2, ndofs; + double w, wq = 0.0; + + dim = el1.GetDim(); + ndof1 = el1.GetDof(); + + nor.SetSize(dim); + + shape1.SetSize(ndof1); + if (Trans.Elem2No >= 0) + { + ndof2 = el2.GetDof(); + shape2.SetSize(ndof2); + } + else + { + ndof2 = 0; + } + + int face_no; + if (ndof2) { face_no = Trans.ElementNo; } + else { face_no = mesh.GetBdrElementFaceIndex(Trans.ElementNo); } + + int info1, info2; + mesh.GetFaceInfos(face_no, &info1, &info2); + int local_face = info1/64; + + const CoarseFineTransformations &cftr = mesh.GetRefinementTransforms(); + + double factor; + bool interior = false; + if (Trans.Elem2No >= 0 && Trans.Elem2No < mesh.GetNE()) + { + int parent_el1 = cftr.embeddings[Trans.Elem1No].parent; + int parent_el2 = cftr.embeddings[Trans.Elem2No].parent; + if (parent_el1 == parent_el2) + { + interior = true; + factor = PenaltyFactor(cftr.embeddings[Trans.Elem1No].matrix, + cftr.embeddings[Trans.Elem2No].matrix); + } + } + if (!interior) + { + factor = kappa*BdrPenaltyFactor(cftr.embeddings[Trans.Elem1No].matrix, + local_face); + } + + ndofs = ndof1 + ndof2; + elmat.SetSize(ndofs); + elmat = 0.0; + + const IntegrationRule *ir = IntRule; + if (ir == NULL) { ir = &IntRules.Get(Trans.GetGeometryType(), 1); } + + for (int q = 0; q < ir->GetNPoints(); q++) + { + const IntegrationPoint &ip = ir->IntPoint(q); + Trans.SetAllIntPoints(&ip); + const IntegrationPoint &eip1 = Trans.GetElement1IntPoint(); + const IntegrationPoint &eip2 = Trans.GetElement2IntPoint(); + + if (dim == 1) { nor(0) = 2*eip1.x - 1.0; } + else { CalcOrtho(Trans.Jacobian(), nor); } + + el1.CalcShape(eip1, shape1); + w = ip.weight; + + double h_face = nor.Norml2(); + double h_el = Trans.Elem1->Weight(); + double h = h_el/h_face; // perpendicular element size + + if (ndof2) + { + el2.CalcShape(eip2, shape2); + double h_el_2 = Trans.Elem2->Weight(); + h = 0.5*h + 0.5*h_el_2/h_face; // average both element sizes + } + + if (interior) + { + wq = w*factor*h_face/h; + } + else + { + wq = w*factor*h_face/h; + } + for (int i = 0; i < ndof1; i++) + { + const double wsi = wq*shape1(i); + for (int j = 0; j < ndof1; j++) + { + elmat(i, j) += wsi * shape1(j); + } + } + if (ndof2) + { + for (int i = 0; i < ndof2; i++) + { + const double wsi = wq*shape2(i); + for (int j = 0; j < ndof1; j++) + { + elmat(ndof1 + i, j) -= wsi * shape1(j); + elmat(j, ndof1 + i) -= wsi * shape1(j); + } + for (int j = 0; j < ndof2; j++) + { + elmat(ndof1 + i, ndof1 + j) += wsi * shape2(j); + } + } + } + } + } +}; + +class DG_LOR_MassPreconditioner : public BilinearFormIntegrator +{ + Mesh &mesh_ho, &mesh_lor; + const int p; + IntegrationRule gll; + Coefficient *Q; + +public: + DG_LOR_MassPreconditioner(Mesh &mesh_ho_, + Mesh &mesh_lor_, + int p_, + Coefficient *Q_) + : mesh_ho(mesh_ho_), + mesh_lor(mesh_lor_), + p(p_), + Q(Q_) + { + QuadratureFunctions1D::GaussLobatto(p+1, &gll); + } + + void AssembleElementMatrix(const FiniteElement &el, + ElementTransformation &Tr, + DenseMatrix &elmat) override + { + const CoarseFineTransformations &cftr = mesh_lor.GetRefinementTransforms(); + const int parent_el = cftr.embeddings[Tr.ElementNo].parent; + // We use the point matrix index to identify the local LOR element index + // within the high-order coarse element. + const int lor_index = cftr.embeddings[Tr.ElementNo].matrix; + + // Assuming piecewise constant + elmat.SetSize(1); + + const int dim = mesh_ho.Dimension(); + IntegrationPoint ip; + if (dim == 2) + { + const int iy = lor_index / (p + 1); + const int ix = lor_index % (p + 1); + ip.x = gll[ix].x; + ip.y = gll[iy].x; + + elmat(0,0) = gll[ix].weight * gll[iy].weight; + } + else if (dim == 3) + { + const int iz = lor_index / (p + 1) / (p + 1); + const int iy = (lor_index / (p + 1)) % (p + 1); + const int ix = lor_index % (p + 1); + + ip.x = gll[ix].x; + ip.y = gll[iy].x; + ip.z = gll[iz].x; + + elmat(0,0) = gll[ix].weight * gll[iy].weight * gll[iz].weight; + } + + ElementTransformation &Tr_ho = *mesh_ho.GetElementTransformation(parent_el); + Tr_ho.SetIntPoint(&ip); + const real_t detJ = Tr_ho.Weight(); + elmat(0,0) *= detJ; + + if (Q) + { + elmat(0,0) *= Q->Eval(Tr_ho, ip); + } + } +}; + +static void TestBatchedLOR_DG(Mesh &mesh, int order) +{ + DG_FECollection fec(order, mesh.Dimension(), BasisType::GaussLobatto); + FiniteElementSpace fespace(&mesh, &fec); + + // Set up some coefficients using H1 grid functions + H1_FECollection h1fec(2, mesh.Dimension()); + FiniteElementSpace h1fes(&mesh, &h1fec); + GridFunction gf1(&h1fes), gf2(&h1fes); + gf1.Randomize(1); + gf2.Randomize(2); + GridFunctionCoefficient mass_coeff(&gf1); + GridFunctionCoefficient diff_coeff(&gf2); + + ConstantCoefficient one(1.0); + constexpr real_t sigma = -1.0; + const int eta = 2; + const int kappa = eta * (order + 1) * (order + 1); + BilinearForm a(&fespace); + a.AddDomainIntegrator(new DiffusionIntegrator); + a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa)); + a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa)); + + Array ess_dofs; // Empty + LORDiscretization lor(fespace); + lor.AssembleSystem(a, ess_dofs); + SparseMatrix &A1 = lor.GetAssembledMatrix(); + + FiniteElementSpace &fes_lor = lor.GetFESpace(); + Mesh &mesh_lor = *fes_lor.GetMesh(); + BilinearForm a_lor(&fes_lor); + a_lor.AddBdrFaceIntegrator(new DG_LOR_DiffusionPreconditioner( + mesh_lor, order, eta)); + a_lor.AddInteriorFaceIntegrator(new DG_LOR_DiffusionPreconditioner( + mesh_lor, order, eta)); + + a_lor.Assemble(); + a_lor.Finalize(); + SparseMatrix &A2 = a_lor.SpMat(); + + TestSameMatrices(A1, A2); +} + +TEST_CASE("LOR Batched DG Orientation", "[LOR][BatchedLOR][CUDA]") +{ + const int order = 3; + const int dim = launch_all_non_regression_tests ? GENERATE(2, 3) : 2; + const int orientation1 = GENERATE_COPY(range(0, dim == 2 ? 4 : 24)); + const int orientation2 = GENERATE_COPY(range(0, dim == 2 ? 4 : 24)); + + CAPTURE(order, dim, orientation1, orientation2); + + Mesh mesh = MeshOrientation(dim, orientation1, orientation2); + TestBatchedLOR_DG(mesh, order); +} + +TEST_CASE("LOR Batched DG", "[LOR][BatchedLOR][CUDA]") +{ + const int order = 3; + const auto mesh_fname = GENERATE( + "../../data/beam-quad.mesh", + "../../data/l-shape.mesh", + "../../data/beam-hex.mesh", + "../../data/fichera.mesh" + ); + CAPTURE(mesh_fname); + Mesh mesh = Mesh::LoadFromFile(mesh_fname); + + mesh.Transform([](const Vector &xin, Vector &xout) + { + for (int d = 0; d < xin.Size(); ++d) + { + xout[d] = xin[d] * (1.0 + d / 3.0); + } + }); + + TestBatchedLOR_DG(mesh, order); +} diff --git a/tests/unit/fem/test_assemblediagonalpa.cpp b/tests/unit/fem/test_pa_diagonal.cpp similarity index 91% rename from tests/unit/fem/test_assemblediagonalpa.cpp rename to tests/unit/fem/test_pa_diagonal.cpp index 620c940b4f..9799aeddbc 100644 --- a/tests/unit/fem/test_assemblediagonalpa.cpp +++ b/tests/unit/fem/test_pa_diagonal.cpp @@ -278,7 +278,7 @@ TEST_CASE("Diffusion Diagonal PA", "[PartialAssembly][AssembleDiagonal]") } template -double test_vdiagpa(int dim, int order) +double test_vdiag_pa(int dim, int order) { Mesh mesh; if (dim == 2) @@ -298,64 +298,43 @@ double test_vdiagpa(int dim, int order) form.AddDomainIntegrator(new INTEGRATOR); form.Assemble(); - Vector diag(fes.GetVSize()); - form.AssembleDiagonal(diag); - BilinearForm form_full(&fes); form_full.AddDomainIntegrator(new INTEGRATOR); form_full.Assemble(); form_full.Finalize(); - Vector diag_full(fes.GetVSize()); + GridFunction x(&fes), y_fa(&fes), y_pa(&fes); + x.Randomize(1); + + form_full.Mult(x, y_fa); + form.Mult(x, y_pa); + y_fa -= y_pa; + REQUIRE(y_fa.Norml2() == MFEM_Approx(0.0)); + + Vector diag(fes.GetVSize()), diag_full(fes.GetVSize()); + form.AssembleDiagonal(diag); form_full.SpMat().GetDiag(diag_full); diag_full -= diag; - return diag_full.Norml2(); } -TEST_CASE("Vector Mass Diagonal PA", "[PartialAssembly][AssembleDiagonal]") +TEST_CASE("Vector Mass Diagonal PA", + "[AssembleDiagonal][PartialAssembly][VectorPA][VectorDiagonalPA][VectorMassPA][CUDA]") { - SECTION("2D") - { - REQUIRE(test_vdiagpa(2, - 2) == MFEM_Approx(0.0)); - - REQUIRE(test_vdiagpa(2, - 3) == MFEM_Approx(0.0)); - } - - SECTION("3D") - { - REQUIRE(test_vdiagpa(3, - 2) == MFEM_Approx(0.0)); - - REQUIRE(test_vdiagpa(3, - 3) == MFEM_Approx(0.0)); - } + const auto DIM = GENERATE(2, 3); + const auto P = GENERATE(1, 2, 3); + CAPTURE(DIM, P); + REQUIRE(test_vdiag_pa(DIM,P) == MFEM_Approx(0.0)); } TEST_CASE("Vector Diffusion Diagonal PA", - "[PartialAssembly][AssembleDiagonal]") + "[AssembleDiagonal][PartialAssembly][VectorPA][VectorDiagonalPA][VectorDiffusionPA][CUDA]") { - SECTION("2D") - { - REQUIRE( - test_vdiagpa(2, - 2) == MFEM_Approx(0.0)); - - REQUIRE(test_vdiagpa(2, - 3) == MFEM_Approx(0.0)); - } - - SECTION("3D") - { - REQUIRE(test_vdiagpa(3, - 2) == MFEM_Approx(0.0)); - - REQUIRE(test_vdiagpa(3, - 3) == MFEM_Approx(0.0)); - } + const auto DIM = GENERATE(2, 3); + const auto P = GENERATE(1, 2, 3); + CAPTURE(DIM, P); + REQUIRE(test_vdiag_pa(DIM,P) == MFEM_Approx(0.0)); } TEST_CASE("Hcurl/Hdiv diagonal PA", diff --git a/tests/unit/fem/test_pa_kernels.cpp b/tests/unit/fem/test_pa_kernels.cpp index 7740b379f2..5991f2bdd2 100644 --- a/tests/unit/fem/test_pa_kernels.cpp +++ b/tests/unit/fem/test_pa_kernels.cpp @@ -9,12 +9,14 @@ // terms of the BSD-3 license. We welcome feedback and contributions, see file // CONTRIBUTING.md for details. +#ifdef _WIN32 +#define _USE_MATH_DEFINES +#include +#endif + #include "unit_tests.hpp" #include "mfem.hpp" -#include -#include - using namespace mfem; namespace pa_kernels @@ -343,60 +345,129 @@ TEST_CASE("Nonlinear Convection", "[PartialAssembly], [NonlinearPA], [GPU]") } template -real_t test_vector_pa_integrator(int dim) +real_t test_pa_vector_integrator(int dim, int sdim) { - Mesh mesh = MakeCartesianNonaligned(dim, 2); - int order = 2; - H1_FECollection fec(order, dim); - FiniteElementSpace fes(&mesh, &fec, dim); + const bool all = launch_all_non_regression_tests; + const auto NE = all ? GENERATE(1, 2, 3) : 2; + const auto p = all ? GENERATE(1, 2, 3): 2; + CAPTURE(p, NE); + + Mesh mesh = MakeCartesianNonaligned(dim, NE); + mesh.SetCurvature(p, false, sdim); + + H1_FECollection fec(p, dim); + FiniteElementSpace fes(&mesh, &fec, sdim); GridFunction x(&fes), y_fa(&fes), y_pa(&fes); x.Randomize(1); + ConstantCoefficient const_coeff(M_PI_2); + FunctionCoefficient funct_coeff([](const Vector &x) { return M_1_PI + x[0]*x[0]; }); + + Vector val(dim); val = 1.0; + VectorConstantCoefficient v_const_coeff(val); + VectorFunctionCoefficient v_funct_coeff(dim, [&](const Vector &x, Vector &v) + { + v(0) = M_LN2 * x(0); + if (dim > 1) { v(1) = M_E * x(1); } + if (dim > 2) { v(2) = M_PI * x(2); } + }); + + MatrixFunctionCoefficient m_funct_coeff(dim, [&](const Vector &x, + DenseMatrix &f) + { + f = 0.0; + if (dim == 1) + { + f(0,0) = 1.1 + sin(M_PI * x[0]); // 1,1 + } + else if (dim == 2) + { + f(0,0) = 1.1 + sin(M_PI * x[1]); // 1,1 + f(1,0) = cos(1.3 * M_PI * x[1]); // 2,1 + f(0,1) = cos(2.5 * M_PI * x[0]); // 1,2 + f(1,1) = 1.1 + sin(4.9 * M_PI * x[0]); // 2,2 + } + else if (dim == 3) + { + f(0,0) = 1.1 + sin(M_PI * x[1]); // 1,1 + f(0,1) = cos(2.5 * M_PI * x[0]); // 1,2 + f(0,2) = sin(4.9 * M_PI * x[2]); // 1,3 + f(1,0) = cos(M_PI * x[0]); // 2,1 + f(1,1) = 1.1 + sin(6.1 * M_PI * x[1]); // 2,2 + f(1,2) = cos(6.1 * M_PI * x[2]); // 2,3 + f(2,0) = sin(1.5 * M_PI * x[1]); // 3,1 + f(2,1) = cos(2.9 * M_PI * x[0]); // 3,2 + f(2,2) = 1.1 + sin(6.1 * M_PI * x[2]); // 3,3 + } + }); + REQUIRE((sdim > dim || m_funct_coeff.GetVDim() == fes.GetVDim())); + BilinearForm blf_fa(&fes); + blf_fa.SetAssemblyLevel(AssemblyLevel::LEGACY); + // scalar coefficients blf_fa.AddDomainIntegrator(new INTEGRATOR); + blf_fa.AddDomainIntegrator(new INTEGRATOR(const_coeff)); + blf_fa.AddDomainIntegrator(new INTEGRATOR(funct_coeff)); + if (sdim == dim) + { + // vector coefficients + blf_fa.AddDomainIntegrator(new INTEGRATOR(v_const_coeff)); + blf_fa.AddDomainIntegrator(new INTEGRATOR(v_funct_coeff)); + // matrix coefficients + blf_fa.AddDomainIntegrator(new INTEGRATOR(m_funct_coeff)); + } blf_fa.Assemble(); blf_fa.Finalize(); blf_fa.Mult(x, y_fa); BilinearForm blf_pa(&fes); blf_pa.SetAssemblyLevel(AssemblyLevel::PARTIAL); + // scalar coefficients blf_pa.AddDomainIntegrator(new INTEGRATOR); + blf_pa.AddDomainIntegrator(new INTEGRATOR(const_coeff)); + blf_pa.AddDomainIntegrator(new INTEGRATOR(funct_coeff)); + if (sdim == dim) + { + // vector coefficients + blf_pa.AddDomainIntegrator(new INTEGRATOR(v_const_coeff)); + blf_pa.AddDomainIntegrator(new INTEGRATOR(v_funct_coeff)); + // matrix coefficients + blf_pa.AddDomainIntegrator(new INTEGRATOR(m_funct_coeff)); + } blf_pa.Assemble(); blf_pa.Mult(x, y_pa); y_fa -= y_pa; - real_t difference = y_fa.Norml2(); - return difference; + return y_fa.Norml2(); } -TEST_CASE("PA Vector Mass", "[PartialAssembly], [VectorPA], [GPU]") +TEST_CASE("PA Vector Mass", + "[PartialAssembly][VectorPA][VectorMassPA][GPU]") { - SECTION("2D") - { - REQUIRE(test_vector_pa_integrator(2) == MFEM_Approx(0.0)); - } - - SECTION("3D") - { - REQUIRE(test_vector_pa_integrator(3) == MFEM_Approx(0.0)); - } + const auto DIM = GENERATE(2, 3); + CAPTURE(DIM); + REQUIRE(test_pa_vector_integrator(DIM, DIM) + == MFEM_Approx(0.0)); } -TEST_CASE("PA Vector Diffusion", "[PartialAssembly], [VectorPA], [GPU]") +TEST_CASE("PA Vector Diffusion", + "[PartialAssembly][VectorPA][VectorDiffusionPA][GPU]") { - SECTION("2D") - { - REQUIRE(test_vector_pa_integrator(2) - == MFEM_Approx(0.0)); - } + const auto DIM = GENERATE(2, 3); + CAPTURE(DIM); + REQUIRE(test_pa_vector_integrator(DIM, DIM) + == MFEM_Approx(0.0)); +} - SECTION("3D") - { - REQUIRE(test_vector_pa_integrator(3) - == MFEM_Approx(0.0)); - } +TEST_CASE("PA Vector Diffusion 2D/3D", + "[PartialAssembly][VectorPA][VectorDiffusionPA][CUDA]") +{ + const int DIM = 2, SDIM = 3; + CAPTURE(DIM, SDIM); + REQUIRE(test_pa_vector_integrator(DIM, SDIM) + == MFEM_Approx(0.0)); } void velocity_function(const Vector &x, Vector &v) diff --git a/tests/unit/general/test_array.cpp b/tests/unit/general/test_array.cpp index 10fdc9a13d..de88b18e28 100644 --- a/tests/unit/general/test_array.cpp +++ b/tests/unit/general/test_array.cpp @@ -124,3 +124,24 @@ TEST_CASE("Array stl-interactions", "[Array]") CHECK(x[i] == y[i]); } } + +TEST_CASE("Array delete at indices", "[Array],[GPU]") +{ + for (int use_dev = 0; use_dev < 2; use_dev++) + { + Array test({0,1,2,3,4,5,6,7,8}); + Array rm_indices({0, 3,4, 6, 8}); + Array result({ 1,2, 5, 7 }); + + test.GetMemory().UseDevice(use_dev); + test.DeleteAt(rm_indices); + + REQUIRE(test.Size() == result.Size()); + + test.HostReadWrite(); + for (int i = 0; i < test.Size(); i++) + { + CHECK(test[i] == result[i]); + } + } +} diff --git a/tests/unit/general/test_scan.cpp b/tests/unit/general/test_scan.cpp new file mode 100644 index 0000000000..c9511bbc00 --- /dev/null +++ b/tests/unit/general/test_scan.cpp @@ -0,0 +1,208 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#include +#include + +#include "mfem.hpp" +#include "unit_tests.hpp" + +// must be included after mfem.hpp +#include "general/scan.hpp" + +using namespace mfem; + +TEST_CASE("Inclusive Scan", "[Scan],[GPU]") +{ + Array a(10); + + for (int use_dev = 0; use_dev < 2; ++use_dev) + { + CAPTURE(use_dev); + a.HostReadWrite(); + for (int i = 0; i < a.Size(); ++i) + { + a[i] = i; + } + auto dptr = a.ReadWrite(use_dev); + InclusiveScan(use_dev, dptr, dptr, a.Size()); + a.HostRead(); + for (int i = 0; i < a.Size(); ++i) + { + int expected = (i + 1) * i / 2; + CAPTURE(i); + REQUIRE(AsConst(a)[i] == expected); + } + a.HostReadWrite(); + for (int i = 0; i < a.Size(); ++i) + { + a[i] = i + 1; + } + a.ReadWrite(use_dev); + InclusiveScan(use_dev, dptr, dptr, a.Size(), std::multiplies<> {}); + a.HostRead(); + int expected = 1; + for (int i = 0; i < a.Size(); ++i) + { + expected *= i + 1; + CAPTURE(i); + REQUIRE(AsConst(a)[i] == expected); + } + } +} + +TEST_CASE("Exclusive Scan", "[Scan],[GPU]") +{ + Array a(10); + + for (int use_dev = 0; use_dev < 2; ++use_dev) + { + CAPTURE(use_dev); + a.HostReadWrite(); + for (int i = 0; i < a.Size(); ++i) + { + a[i] = i; + } + auto dptr = a.ReadWrite(use_dev); + ExclusiveScan(use_dev, dptr, dptr, a.Size(), 5); + a.HostRead(); + for (int i = 0; i < a.Size(); ++i) + { + int expected = (i + 1) * i / 2 - i + 5; + CAPTURE(i); + REQUIRE(AsConst(a)[i] == expected); + } + a.HostReadWrite(); + for (int i = 0; i < a.Size(); ++i) + { + a[i] = i + 1; + } + a.ReadWrite(use_dev); + ExclusiveScan(use_dev, dptr, dptr, a.Size(), 5, std::multiplies<> {}); + a.HostRead(); + int expected = 5; + for (int i = 0; i < a.Size(); ++i) + { + CAPTURE(i); + REQUIRE(AsConst(a)[i] == expected); + expected *= i + 1; + } + } +} + +TEST_CASE("CopyFlagged", "[Scan],[GPU]") +{ + Array a(10); + Array flags(a.Size()); + Array res(a.Size()); + Array num_selected_out(1); + + for (int use_dev = 0; use_dev < 2; ++use_dev) + { + CAPTURE(use_dev); + a.HostWrite(); + for (int i = 0; i < a.Size(); ++i) + { + a[i] = i; + } + flags.HostWrite(); + // keep entries which are a multiple of 3 + for (int i = 0; i < flags.Size(); ++i) + { + if (i % 3) + { + flags[i] = false; + } + else + { + flags[i] = true; + } + } + auto d_in = a.Read(use_dev); + auto d_flags = flags.Read(use_dev); + auto d_out = res.Write(use_dev); + auto d_num_selected_out = num_selected_out.Write(use_dev); + CopyFlagged(use_dev, d_in, d_flags, d_out, d_num_selected_out, a.Size()); + res.HostRead(); + num_selected_out.HostRead(); + REQUIRE(AsConst(num_selected_out)[0] == 4); + REQUIRE(AsConst(res)[0] == 0); + REQUIRE(AsConst(res)[1] == 3); + REQUIRE(AsConst(res)[2] == 6); + REQUIRE(AsConst(res)[3] == 9); + } +} + +TEST_CASE("CopyIf", "[Scan][GPU]") +{ + Array a(10); + Array res(a.Size()); + Array num_selected_out(1); + + for (int use_dev = 1; use_dev < 2; ++use_dev) + { + CAPTURE(use_dev); + a.HostWrite(); + res.HostWrite(); + num_selected_out.HostWrite(); + num_selected_out[0] = 0; + for (int i = 0; i < a.Size(); ++i) + { + a[i] = i; + res[i] = 0; + } + auto d_in = a.Read(use_dev); + auto d_out = res.Write(use_dev); + auto d_num_selected_out = num_selected_out.Write(use_dev); + // copy all values not divisible by 3 + // 1, 2, 4, 5, 7, 8 + CopyIf(use_dev, d_in, d_out, d_num_selected_out, a.Size(), + [=] MFEM_HOST_DEVICE(const int &value) { return value % 3; }); + res.HostRead(); + num_selected_out.HostRead(); + REQUIRE(AsConst(num_selected_out)[0] == 6); + REQUIRE(AsConst(res)[0] == 1); + REQUIRE(AsConst(res)[1] == 2); + REQUIRE(AsConst(res)[2] == 4); + REQUIRE(AsConst(res)[3] == 5); + REQUIRE(AsConst(res)[4] == 7); + REQUIRE(AsConst(res)[5] == 8); + } +} + +TEST_CASE("CopyUnique", "[Scan][GPU]") +{ + Array a(10); + Array res(a.Size()); + Array num_selected_out(1); + + for (int use_dev = 0; use_dev < 2; ++use_dev) + { + CAPTURE(use_dev); + a.HostWrite(); + for (int i = 0; i < a.Size(); ++i) + { + a[i] = 2; + } + a[4] = 1; + a[5] = 1; + auto d_in = a.Read(use_dev); + auto d_out = res.Write(use_dev); + auto d_num_selected_out = num_selected_out.Write(use_dev); + CopyUnique(use_dev, d_in, d_out, d_num_selected_out, a.Size()); + res.HostRead(); + num_selected_out.HostRead(); + REQUIRE(AsConst(num_selected_out)[0] == 3); + REQUIRE(AsConst(res)[0] == 2); + REQUIRE(AsConst(res)[1] == 1); + REQUIRE(AsConst(res)[2] == 2); + } +} diff --git a/tests/unit/linalg/test_matrix_dense.cpp b/tests/unit/linalg/test_matrix_dense.cpp index 4a1198c5ce..0e3422711d 100644 --- a/tests/unit/linalg/test_matrix_dense.cpp +++ b/tests/unit/linalg/test_matrix_dense.cpp @@ -373,6 +373,7 @@ TEST_CASE("Batched Linear Algebra", const int n_rhs = 2; DenseTensor A_batch(n, n, n_mat); + DenseTensor A_inv_batch(n, n, n_mat); Vector x_batch(n * n_rhs * n_mat), y_batch(n * n_rhs * n_mat); std::vector As; std::vector xs, ys; @@ -404,6 +405,7 @@ TEST_CASE("Batched Linear Algebra", ys.back() = 0.0; AddMult_a(1.5, As.back(), xs.back(), ys.back()); A_batch(i) = As.back(); + A_inv_batch(i) = As.back(); } // Test batched matrix-vector products @@ -463,6 +465,33 @@ TEST_CASE("Batched Linear Algebra", } } } + + // Test batched matrix inverse + BatchedLinAlg::Get(backend).Invert(A_inv_batch); + A_inv_batch.HostReadWrite(); + Vector output_col(n); + Vector col; + for (int i = 0; i < n_mat; ++i) + { + DenseMatrix Ai_inv(A_inv_batch(i)); + for (int j = 0; j < n; ++j) + { + output_col = 0.0; + As[i].GetColumnReference(j, col); + Ai_inv.Mult(col, output_col); + for (int k = 0; k < n; ++k) + { + if (j == k) + { + REQUIRE(output_col(k) == MFEM_Approx(1.0)); + } + else + { + REQUIRE(output_col(k) == MFEM_Approx(0.0)); + } + } + } + } } TEST_CASE("DenseTensor copy", "[DenseMatrix][DenseTensor]") diff --git a/tests/unit/linalg/test_matrix_sparse.cpp b/tests/unit/linalg/test_matrix_sparse.cpp index 219bc63df4..88feb4f85f 100644 --- a/tests/unit/linalg/test_matrix_sparse.cpp +++ b/tests/unit/linalg/test_matrix_sparse.cpp @@ -240,4 +240,54 @@ TEST_CASE("SparseMatrix printing", "[SparseMatrix]") } } +TEST_CASE("SparseMatrix cuSPARSE Bug", "[SparseMatrix][GPU]") +{ + // This test case ensures that we have a functioning workaround for the bug + // CUSPARSE-1897. In versions of cuSPARSE before 12.8, the internal buffer + // used for cusparseSpMV must be the same when it is called with the same + // matrix. + // + // By default, MFEM uses one buffer, that is shared by all sparse matrices. + // In the code below, a buffer is created for A, then modified for B, then + // used again for A. Without the workaround, this fails with cuSPARSE version + // earlier than 12.8 (confirmed to fail with 12.4). + + const int n = 100; + SparseMatrix A(n, n); + Vector d(n); + d.Randomize(1); + for (int i = 0; i < n; ++i) + { + A.Set(i, i, d[i]); + } + A.Finalize(); + + Vector x(n); + x = 1.0; + + Vector y(n); + A.Mult(x, y); + + { + SparseMatrix B(20, 20); + for (int i = 0; i < 20; ++i) + { + for (int j = 0; j < 20; ++j) + { + B.Set(i, j, 1.0); + } + } + B.Finalize(); + Vector u(20); + u = 1.0; + Vector v(20); + B.Mult(u, v); + } + + A.Mult(x, y); + + y -= d; + REQUIRE(y.Normlinf() == MFEM_Approx(0.0)); +} + } // namespace mfem diff --git a/tests/unit/linalg/test_same_matrices.hpp b/tests/unit/linalg/test_same_matrices.hpp new file mode 100644 index 0000000000..d435dd081b --- /dev/null +++ b/tests/unit/linalg/test_same_matrices.hpp @@ -0,0 +1,97 @@ +// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced +// at the Lawrence Livermore National Laboratory. All Rights reserved. See files +// LICENSE and NOTICE for details. LLNL-CODE-806117. +// +// This file is part of the MFEM library. For more information and source code +// availability visit https://mfem.org. +// +// MFEM is free software; you can redistribute it and/or modify it under the +// terms of the BSD-3 license. We welcome feedback and contributions, see file +// CONTRIBUTING.md for details. + +#ifndef MFEM_TEST_SAME_MATRICES_HPP +#define MFEM_TEST_SAME_MATRICES_HPP + +#include "mfem.hpp" +#include "unit_tests.hpp" + +namespace mfem +{ + +#ifndef MFEM_USE_MPI +#define HYPRE_BigInt int +#endif // MFEM_USE_MPI + +inline void TestSameMatrices( + SparseMatrix &A1, const SparseMatrix &A2, + HYPRE_BigInt *cmap1=nullptr, + std::unordered_map *cmap2inv=nullptr) +{ + REQUIRE(A1.Height() == A2.Height()); + int n = A1.Height(); + + const int *I1 = A1.HostReadI(); + const int *J1 = A1.HostReadJ(); + const real_t *V1 = A1.HostReadData(); + + A2.HostReadI(); + A2.HostReadJ(); + A2.HostReadData(); + + real_t error = 0.0; + + for (int i=0; icount(cmap1[j]) > 0) + { + j = (*cmap2inv)[cmap1[j]]; + } + else + { + error = std::max(error, std::fabs(V1[jj])); + continue; + } + } + error = std::max(error, std::fabs(V1[jj] - A2(i,j))); + } + } + + REQUIRE(error == MFEM_Approx(0.0, 1e-10)); +} + +#ifdef MFEM_USE_MPI + +inline void TestSameMatrices(HypreParMatrix &A1, const HypreParMatrix &A2) +{ + HYPRE_BigInt *cmap1, *cmap2; + SparseMatrix diag1, offd1, diag2, offd2; + + A1.GetDiag(diag1); + A2.GetDiag(diag2); + A1.GetOffd(offd1, cmap1); + A2.GetOffd(offd2, cmap2); + + TestSameMatrices(diag1, diag2); + + if (cmap1) + { + std::unordered_map cmap2inv; + for (int i=0; i rm_indices({0, 3,4, 6, 8}); + Vector result({ 1,2, 5, 7 }); + + test.UseDevice(use_dev); + test.DeleteAt(rm_indices); + + REQUIRE(test.Size() == result.Size()); + + test.HostReadWrite(); + for (int i = 0; i < test.Size(); i++) + { + CHECK(test[i] == result[i]); + } + } +} diff --git a/tests/unit/mesh/test_nurbs.cpp b/tests/unit/mesh/test_nurbs.cpp index dc5b317ff1..1267a4f08e 100644 --- a/tests/unit/mesh/test_nurbs.cpp +++ b/tests/unit/mesh/test_nurbs.cpp @@ -131,3 +131,33 @@ TEST_CASE("NURBS mesh reconstruction", "[NURBS]") // Cleanup for (auto *p : patches) { delete p; } } + +TEST_CASE("NURBS NC-patch mesh loading", "[NURBS]") +{ + auto mesh_fname = GENERATE("../../data/nc3-nurbs.mesh", + "../../data/nc-nurbs3d.mesh"); + + Mesh mesh(mesh_fname, 1, 1); + const int dim = mesh.Dimension(); + const int ne = dim == 2 ? 6 : 24; + REQUIRE(mesh.GetNE() == ne); + + mesh.NURBSUniformRefinement(); + REQUIRE(mesh.GetNE() == ne * std::pow(2, dim)); +} + +TEST_CASE("NURBS NC-patch large meshes", "[MFEMData][NURBS]") +{ + auto mesh_fname = GENERATE("bricks2D.mesh", + "schwarz2D.mesh", + "schwarz3D.mesh"); + + const std::string & fpath = (mfem_data_dir + "/nurbs/nc_patch/"); + + Mesh mesh(fpath + mesh_fname, 1, 1); + const int dim = mesh.Dimension(); + const int ne = mesh.GetNE(); + + mesh.NURBSUniformRefinement(); + REQUIRE(mesh.GetNE() == ne * std::pow(2, dim)); +}