Add more annotations, modifying LOR solvers miniapp for benchmarking

[Github] Sanitizers action

.github/actions/sanitize/config/action.yml

@@ -0,0 +1,154 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: Sanitizer Config
+description: Sets up environment variables for MFEM sanitizer workflow
+
+inputs:
+  DEBUG:
+    description: If true, use intermediate caches to speed up the workflow
+                 by reusing previous builds.
+    default: false
+
+  REPOSITORY:
+    description: Repository to checkout
+    default: mfem/mfem
+
+  BRANCH:
+    description: Branch to checkout
+    default: ubsan
+
+  CLANG_VER:
+    description: CLANG version to use
+    default: 18
+
+  # https://github.com/llvm/llvm-project/releases
+  LLVM_VER:
+    description: LLVM version to use
+    default: 19.1.7
+
+  # https://github.com/hypre-space/hypre/releases
+  HYPRE_VER:
+    description: HYPRE version to use
+    default: 2.19.0
+
+  METIS_VER:
+    description: METIS version to use
+    default: 4.0.3
+
+  CTEST:
+    description: CTest command to use
+    default: ctest -j --test-load $(nproc)
+                      --schedule-random
+                      --stop-on-failure --output-on-failure
+                      --test-dir
+
+  # https://clang.llvm.org/docs/AddressSanitizer.html
+  ASAN_OPTIONS:
+    default: detect_leaks=1,
+             strict_init_order=1,
+             strict_string_checks=1,
+             check_initialization_order=1,
+             detect_stack_use_after_return=1
+  ASAN_CXXFLAGS:
+    default: -fsanitize=address
+             -fsanitize-address-use-after-scope
+  ASAN_LDFLAGS:
+    default: -fsanitize=address
+
+  # https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
+  UBSAN_OPTIONS:
+    default: halt_on_error=1, print_stacktrace=1
+  UBSAN_CXXFLAGS:
+    default: -fsanitize=undefined
+  UBSAN_LDFLAGS:
+    default: -fsanitize=undefined
+
+  # https://clang.llvm.org/docs/MemorySanitizer.html
+  MSAN_OPTIONS:
+    default: "poison_in_dtor=1"
+  MSAN_CXXFLAGS:
+    default: -fsanitize=memory
+             -fsanitize-memory-track-origins
+             -fsanitize-memory-use-after-dtor
+  MSAN_LDFLAGS:
+    default: -fsanitize=memory
+
+  LSAN_DIR:
+    description: LSAN suppression directory
+    default: lsan
+
+  LSAN_FILE:
+    description: LSAN suppression file
+    default: lsan.supp
+
+  NO_FLAGS:
+    description: If true, do not set any CXXFLAGS or LDFLAGS.
+    default: false
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Env (Inputs)
+      run: |
+        echo DEBUG=${{inputs.DEBUG}} >> $GITHUB_ENV
+        echo REPOSITORY=${{inputs.REPOSITORY}} >> $GITHUB_ENV
+        echo BRANCH=${{inputs.BRANCH}} >> $GITHUB_ENV
+        echo CLANG_VER=${{inputs.CLANG_VER}} >> $GITHUB_ENV
+        echo LLVM_VER=${{inputs.LLVM_VER}} >> $GITHUB_ENV
+        echo HYPRE_VER=${{inputs.HYPRE_VER}} >> $GITHUB_ENV
+        echo METIS_VER=${{inputs.METIS_VER}} >> $GITHUB_ENV
+        echo CTEST=${{inputs.CTEST}} >> $GITHUB_ENV
+        echo ASAN_OPTIONS=${{inputs.ASAN_OPTIONS}} >> $GITHUB_ENV
+        echo UBSAN_OPTIONS=${{inputs.UBSAN_OPTIONS}} >> $GITHUB_ENV
+        echo MSAN_OPTIONS=${{inputs.MSAN_OPTIONS}} >> $GITHUB_ENV
+        echo LSAN_DIR=${{inputs.LSAN_DIR}} >> $GITHUB_ENV
+        echo LSAN_FILE=${{inputs.LSAN_FILE}} >> $GITHUB_ENV
+        echo ASAN_CXXFLAGS=${{inputs.ASAN_CXXFLAGS}} >> $GITHUB_ENV
+        echo ASAN_LDFLAGS=${{inputs.ASAN_LDFLAGS}} >> $GITHUB_ENV
+        echo UBSAN_CXXFLAGS=${{inputs.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
+        echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
+        echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
+        echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (dir)
+      run: |
+        echo LLVM_DIR=${{github.workspace}}/llvm >> $GITHUB_ENV
+        echo HYPRE_DIR=hypre-${{inputs.HYPRE_VER}} >> $GITHUB_ENV
+        echo METIS_DIR=metis-${{inputs.METIS_VER}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (bis)
+      run: |
+        echo CC=clang-${{inputs.CLANG_VER}} >> $GITHUB_ENV
+        echo CXX=clang++-${{inputs.CLANG_VER}} >> $GITHUB_ENV
+        echo LLVM_INC=${{env.LLVM_DIR}}/include/c++/v1 >> $GITHUB_ENV
+        echo LLVM_LIB=${{env.LLVM_DIR}}/lib >> $GITHUB_ENV
+        echo HYPRE_TGZ=v${{inputs.HYPRE_VER}}.tar.gz >> $GITHUB_ENV
+        echo METIS_TGZ=metis-${{inputs.METIS_VER}}.tar.gz >> $GITHUB_ENV
+        LSAN_SUPPRESSIONS="${{github.workspace}}/${{inputs.LSAN_DIR}}/${{inputs.LSAN_FILE}}"
+        echo "LSAN_OPTIONS=suppressions=$LSAN_SUPPRESSIONS" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (ter)
+      if: ${{ inputs.NO_FLAGS != 'true' }}
+      run: |
+        echo LLVM_CXXFLAGS=-stdlib=libc++ -I${{env.LLVM_INC}} -Isystem${{env.LLVM_INC}} >> $GITHUB_ENV
+        echo LLVM_LDFLAGS=-L${{env.LLVM_LIB}} -lc++abi -Wl,-rpath,${{env.LLVM_LIB}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (quater)
+      if: ${{ inputs.NO_FLAGS != 'true' }}
+      run: |
+        echo CXXFLAGS=${{env.LLVM_CXXFLAGS}} >> $GITHUB_ENV
+        echo LDFLAGS=${{env.LLVM_LDFLAGS}} >> $GITHUB_ENV
+      shell: bash

.github/actions/sanitize/mfem/action.yml

@@ -0,0 +1,91 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'MFEM Compilation'
+description: 'MFEM Compilation'
+
+inputs:
+  par:
+    description: 'Whether to build for parallel (true/false)'
+    default: false
+  sanitizer:
+    description: 'Sanitizer to use (asan, msan, ubsan)'
+    default: asan
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: ./.github/actions/sanitize/config
+
+    - uses: actions/cache@v4
+      if: ${{env.DEBUG == 'true'}}
+      id: debug
+      with:
+        path: mfem/build
+        key: build-${{inputs.par}}-${{inputs.sanitizer}}
+
+    - uses: ./.github/actions/sanitize/setup
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      with:
+        par: ${{inputs.par}}
+        sanitizer: ${{inputs.sanitizer}}
+
+    - name: Build with ASAN
+      if: inputs.sanitizer == 'asan'
+      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.ASAN_CXXFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Build with MSAN
+      if: inputs.sanitizer == 'msan'
+      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MSAN_CXXFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Build with UBSAN
+      if: inputs.sanitizer == 'ubsan'
+      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - uses: mfem/github-actions/build-mfem@v2.5
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      env:
+        CXXFLAGS: ${{env.CXXFLAGS}}
+        LDFLAGS: ${{env.LDFLAGS}}
+      with:
+        mpi: ${{inputs.par == 'false' && 'seq' || 'par'}}
+        mfem-dir: mfem
+        os: ${{runner.os}}
+        library-only: true
+        build-system: cmake
+        hypre-dir: ${{env.HYPRE_DIR}}
+        metis-dir: ${{env.METIS_DIR}}
+        config-options: >-
+          -GNinja
+          -DMPICXX=${{env.CXX}}
+          -DCMAKE_CXX_STANDARD=17
+          -DMFEM_USE_MEMALLOC=OFF
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_VERBOSE_MAKEFILE=ON
+          -DCMAKE_CXX_COMPILER=${{env.CXX}}
+          -DCMAKE_CXX_FLAGS_RELEASE='-g -O1 -fno-omit-frame-pointer'
+
+    - name: Delete object files
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      working-directory: mfem/build
+      run: find . -type f -name '*.o' -delete
+      shell: bash
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: build-${{inputs.par}}-${{inputs.sanitizer}}
+        path: mfem/build
+        if-no-files-found: error
+        retention-days: 1
+        overwrite: false

.github/actions/sanitize/mpi/action.yml

@@ -0,0 +1,33 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'Install MPI'
+description: 'Installs MPI and set up its environment variables'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install
+      run: sudo apt-get install openmpi-bin libopenmpi-dev
+      shell: bash
+
+    - name: Env
+      run: |
+        echo PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe >> $GITHUB_ENV
+        echo MPI_INC=$(mpicxx --showme:compile) >> $GITHUB_ENV
+        echo MPI_LIB=$(mpicxx --showme:link) >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (bis)
+      run: |
+        echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
+        echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV
+      shell: bash

.github/actions/sanitize/restore/action.yml

@@ -0,0 +1,71 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'Restore state'
+description: 'Restore state to be able to run checks, tests'
+
+inputs:
+  par:
+    description: 'Whether to build for parallel (true/false)'
+    default: false
+  sanitizer:
+    description: 'Sanitizer to use (asan, msan, ubsan)'
+    default: asan
+  cache-path:
+    description: 'path to what needs to be restored'
+    default: none
+  cache-skip:
+    description: 'Skip cache restoration'
+    default: false
+
+outputs:
+  cache-hit:
+    description: 'Output from a specific step'
+    value: ${{steps.debug.outputs.cache-hit}}
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: ./.github/actions/sanitize/config
+
+    - uses: actions/cache@v4
+      if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
+      id: debug
+      with:
+        path: ${{inputs.cache-path}}
+        key: ${{github.job}}-${{inputs.par}}-${{inputs.sanitizer}}
+
+    - uses: ./.github/actions/sanitize/setup
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      with:
+        par: ${{inputs.par}}
+        sanitizer: ${{inputs.sanitizer}}
+
+    - uses: actions/download-artifact@v4
+      with:
+        name: build-${{inputs.par}}-${{inputs.sanitizer}}
+        path: mfem/build
+
+    - name: Ninja Patch
+      working-directory: mfem/build
+      run: |
+        sed -i -e 's/CXX_STATIC_LIBRARY_LINKER__mfem_Release.*/CUSTOM_COMMAND/' build.ninja
+        sed -i -e '/build tests\/unit\/all:/ s/tests\/unit\/[^ ]*unit_tests[^ ]*//g' build.ninja
+        sed -i -e '/^add_test(\[=\[\(unit_tests\|punit_tests\)\]=\]/ s/)/ "--input-file .\/list-test-names-${{matrix.tag}}" "--min-duration 1")/' tests/unit/CTestTestfile.cmake
+      shell: bash
+
+    - name: Copy Data
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      working-directory: mfem/build
+      run: |
+        ninja cmake_object_order_depends_target_unit_tests
+        cp -pR ../tests/unit/data tests/unit
+      shell: bash

.github/actions/sanitize/setup/action.yml

@@ -0,0 +1,64 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'Setup state'
+description: 'Sets up the state to be able to run build & run'
+
+inputs:
+  par:
+    description: 'Whether to build for parallel (true/false)'
+    default: false
+  sanitizer:
+    description: 'Sanitizer to use (asan, msan, ubsan)'
+    default: asan
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/cache/restore@v4  # Cache for LLVM libcxx
+      with:
+        path: ${{env.LLVM_DIR}}
+        fail-on-cache-miss: true
+        key: build-libcxx-${{env.LLVM_VER}}-${{inputs.sanitizer}}
+
+    - uses: ./.github/actions/sanitize/mpi
+      if: ${{inputs.par == 'true'}}
+
+    - uses: actions/cache/restore@v4  # Cache for Hypre
+      if: ${{inputs.par == 'true'}}
+      with:
+        path: ${{env.HYPRE_DIR}}
+        fail-on-cache-miss: true
+        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+
+    - uses: actions/cache/restore@v4  # Cache for Metis
+      if: ${{inputs.par == 'true'}}
+      with:
+        path: ${{env.METIS_DIR}}
+        fail-on-cache-miss: true
+        key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+
+    - name: Hypre/Metis links
+      if: ${{inputs.par == 'true'}}
+      run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
+      shell: bash
+
+    - uses: actions/cache/restore@v4  # Cache for LSAN suppression file
+      with:
+        path: ${{env.LSAN_DIR}}
+        fail-on-cache-miss: true
+        key: build-lsan-suppression-file
+
+    - uses: actions/checkout@v4  # Checkout the repository
+      with:
+        path: mfem
+        # ref: ${{env.BRANCH}}
+        # repository: ${{env.REPOSITORY}}

.github/workflows/README.md

@@ -7,18 +7,17 @@
 
                                https://mfem.org
 
-
 This directory contains the GitHub CI scripts for MFEM.
 
 Note that some of these scripts use the shared MFEM GitHub Actions from the external mfem/github-actions repository:
 
-  https://github.com/mfem/github-actions
+  <https://github.com/mfem/github-actions>
 
-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.1`, the `v2.1` suffix denotes the branch in the above from which the action is taken.
+For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
 
 The current CI workflows are:
 
-### `repo-check.yml`
+## `repo-check.yml`
 
 Runs a number of static repository-level sanity checks.
 
@@ -30,19 +29,39 @@ Runs a number of static repository-level sanity checks.
 
 - `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
 
-### `mfem-analysis.yml` (`build-analysis`)
+## `mfem-analysis.yml` (`build-analysis`)
 
 Checks if the code builds and satisfies minimal requirements.
 
 - `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
 
-### `builds-and-tests.yml`
+## `builds-and-tests.yml`
 
 Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
 
-Uses the following GitHub Actions from https://github.com/mfem/github-actions:
+Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
 
 - `mfem/github-actions/build-hypre`
 - `mfem/github-actions/build-metis`
 - `mfem/github-actions/build-mfem`
 - `mfem/github-actions/upload-coverage`
+
+## Sanitizer Workflow for MFEM Verification
+
+This workflow validates MFEM unit tests, examples, and miniapps using sanitizer tools.
+
+- `sanitizers.yml` orchestrates:
+  - Building and caching dependencies: HYPRE, METIS, LSAN suppression file, and LLVM libcxx.
+  - Launching fine-grained jobs for serial (ASAN, MSAN, UBSAN) and parallel (ASAN, UBSAN) sanitizers.
+- `sanitize-tests.yml` is a reusable workflow accepting `par` mode (`true` for parallel) and `sanitizer` (ASAN, MSAN, or UBSAN) as inputs. It executes the following jobs:
+  - **Build**: Compiles the MFEM library with specified parallel and sanitizer settings.
+  - **Check**: Runs verification checks.
+  - Parallel jobs to test the following: **Examples**, **Miniapps** and **Unit tests**
+
+The workflow leverages composite actions in `.github/actions/sanitize/`:
+
+- `config`: Centralizes settings for the sanitizer workflow.
+- `mfem`: Manages the MFEM library build process.
+- `mpi`: Installs MPI and applies additional compilation flags.
+- `restore`: Restores the testing environment state.
+- `setup`: Builds or restores cached dependencies.

.github/workflows/mfem-sanitizer.yml

@@ -1,69 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Sanitizer"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  Serial:
-    runs-on: ubuntu-24.04
-
-    steps:
-      - name: MFEM Checkout
-        uses: actions/checkout@v4
-        with:
-          path: mfem
-
-      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.5
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          mpi: seq
-          hypre-dir: unused-hypre-dir
-          metis-dir: unused-metis-dir
-          mfem-dir: mfem
-          build-system: make
-          library-only: false
-          config-options:
-            CXX="clang++-18"
-            CXXFLAGS="-g -O1 -std=c++17
-                      -fsanitize=address
-                      -fno-omit-frame-pointer
-                      -fsanitize-address-use-after-scope"
-
-      - name: MFEM Info
-        working-directory: mfem
-        run: make info
-
-      - name: MFEM Sanitize
-        working-directory: mfem
-        run:
-          ASAN_OPTIONS="detect_leaks=1,
-                        strict_init_order=1,
-                        strict_string_checks=1,
-                        check_initialization_order=1,
-                        detect_stack_use_after_return=1"
-          make test

.github/workflows/sanitize-build-hypre.yml

@@ -0,0 +1,39 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-hypre
+on:
+  workflow_call:
+jobs:
+  build-hypre:
+    runs-on: ubuntu-latest
+    name: 2.19.0
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.HYPRE_DIR}}
+          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+      - name: Setup
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/sanitize/mpi
+      - name: Build
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-hypre@v2.5
+        with:
+          archive: ${{env.HYPRE_TGZ}}
+          dir: ${{env.HYPRE_DIR}}
+          target: int32
+          precision: fp64
+          build-system: make

.github/workflows/sanitize-build-libcxx.yml

@@ -0,0 +1,76 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-libcxx
+on:
+  workflow_call:
+jobs:
+  build-llvm-libcxx:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        sanitizer: [asan, msan, ubsan]
+        include:
+          - sanitizer: asan
+            llvm_use_sanitizer: "Address"
+          - sanitizer: msan
+            llvm_use_sanitizer: "MemoryWithOrigins"
+          - sanitizer: ubsan
+            llvm_use_sanitizer: "Undefined"
+    name: ${{matrix.sanitizer}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+        with:
+          NO_FLAGS: true
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.LLVM_DIR}}
+          key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}
+      - name: Clone
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: >
+          git clone --filter=blob:none --depth=1
+          --branch llvmorg-${{env.LLVM_VER}}
+          --no-checkout https://github.com/llvm/llvm-project.git llvm-project
+      - name: Checkout
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        working-directory: llvm-project
+        run: |
+          git sparse-checkout set --cone
+          git checkout llvmorg-${{env.LLVM_VER}}
+          git sparse-checkout set cmake llvm/cmake runtimes libcxx libcxxabi
+      - name: Mkdir
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: mkdir ${{env.LLVM_DIR}}
+      - name: CMake
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        working-directory: ${{env.LLVM_DIR}}
+        run: >
+          VERBOSE=1
+          cmake -GNinja ../llvm-project/runtimes/
+          -DCMAKE_C_COMPILER=${{env.CC}}
+          -DCMAKE_CXX_COMPILER=${{env.CXX}}
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo
+          -DCMAKE_INSTALL_PREFIX=/usr
+          -DLLVM_USE_SANITIZER=${{matrix.llvm_use_sanitizer}}
+          -DLLVM_BUILD_32_BITS=OFF
+          -DLIBCXXABI_USE_LLVM_UNWINDER=OFF
+          -DLLVM_INCLUDE_TESTS=OFF
+          -DLIBCXX_INCLUDE_TESTS=OFF
+          -DLIBCXX_INCLUDE_BENCHMARKS=OFF
+          -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi'
+      - name: Build
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        working-directory: ${{env.LLVM_DIR}}
+        run: cmake --build . -- cxx cxxabi

.github/workflows/sanitize-build-lsan.yml

@@ -0,0 +1,38 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-file-lsan
+on:
+  workflow_call:
+jobs:
+  build-file-lsan:
+    runs-on: ubuntu-latest
+    name: lsan.supp
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.LSAN_DIR}}
+          key: build-lsan-suppression-file
+      - name: Setup
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p ${{env.LSAN_DIR}}
+          cat << EOF > ${{env.LSAN_DIR}}/${{env.LSAN_FILE}}
+          leak:libevent_core-2.1.so
+          leak:ompi_mpi_finalize
+          leak:ompi_mpi_init
+          leak:PMPI_Init
+          leak:strdup
+          EOF

.github/workflows/sanitize-build-metis.yml

@@ -0,0 +1,36 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-metis
+on:
+  workflow_call:
+jobs:
+  build-metis:
+    runs-on: ubuntu-latest
+    name: 4.0.3
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.METIS_DIR}}
+          key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+      - name: Setup
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/sanitize/mpi
+      - name: Build
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-metis@v2.5
+        with:
+          archive: ${{env.METIS_TGZ}}
+          dir: ${{env.METIS_DIR}}

.github/workflows/sanitize-tests.yml

@@ -0,0 +1,197 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: Sanitize
+on:
+  workflow_call:
+    inputs:
+      par:
+        description: 'Whether to build for parallel (true/false)'
+        required: false
+        default: false
+        type: boolean
+      sanitizer:
+        description: 'Sanitizer to use (asan, msan, ubsan)'
+        required: true
+        default: asan
+        type: string
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/mfem
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+
+  check:
+    needs: [build]
+    runs-on: ubuntu-latest
+    env:
+      ex: ${{inputs.par && 'ex1p' || 'ex1'}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/examples/${{env.ex}}
+      - name: MFEM Check
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v check
+
+  examples:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      exclude: ${{inputs.par && '-E "_ser"' || ''}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/examples/ex1
+      - name: Build Examples
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v examples
+      - name: Test Examples
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} examples ${{env.exclude}} --show-only
+          ${{env.CTEST}} examples ${{env.exclude}}
+
+  miniapps:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      exclude: ${{inputs.par && '-E "_ser"' || ''}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/miniapps/meshing/minimal-surface
+      - name: Build Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v miniapps
+      - name: Test Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} miniapps ${{env.exclude}} --show-only
+          ${{env.CTEST}} miniapps ${{env.exclude}}
+
+  tests-miniapps:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      run: ${{inputs.par && '-R "_cpu_np"' || ''}}
+      exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/tests/unit/sedov_tests_cpu
+      - name: Build Tests Unit Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v tests/unit/all
+      - name: Run Tests Unit Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}} --show-only
+          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}}
+
+  tests-unit-build:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
+      - name: Build Unit Tests
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v ${{env.unit_tests}}
+      - name: Delete object files
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build/tests/unit
+        run: find . -type f -name '*.o' -delete
+      - uses: actions/upload-artifact@v4
+        with:
+          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
+          path: mfem/build/tests/unit/${{env.unit_tests}}
+          if-no-files-found: error
+          retention-days: 1
+          overwrite: false
+
+  tests-unit-run:
+    needs: [tests-unit-build]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tag: [0, 1, 2, 3]
+    name: tests-unit-run-${{matrix.tag}}
+    env:
+      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
+      np: ${{inputs.par && '_np=2' || ''}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
+      - uses: actions/download-artifact@v4
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        with:
+          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
+          path: mfem/build/tests/unit
+      - name: Split Unit Tests
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build/tests/unit
+        run: |
+          chmod 755 ${{env.unit_tests}}
+          ./${{env.unit_tests}} --list-test-names-only | tail -n +2 > list-test-names
+          shuf list-test-names -o list-test-names
+          split --verbose -n l/4 -d -a 1 list-test-names list-test-names-
+      - name: Cat Unit Tests ${{matrix.tag}}
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build/tests/unit
+        run: cat list-test-names-${{matrix.tag}}
+      - name: Run Unit Tests ${{matrix.tag}}
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}" --show-only
+          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}"

.github/workflows/sanitizers.yml

@@ -0,0 +1,73 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: Sanitizers
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches: ["master", "next"]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+
+  # Build steps for dependencies
+  build-hypre:
+    uses: ./.github/workflows/sanitize-build-hypre.yml
+
+  build-metis:
+    uses: ./.github/workflows/sanitize-build-metis.yml
+
+  build-lsan:
+    uses: ./.github/workflows/sanitize-build-lsan.yml
+
+  build-libcxx:
+    uses: ./.github/workflows/sanitize-build-libcxx.yml
+
+  # Serial sanitizers: asan, msan, ubsan
+  seq-asan:
+    needs: [build-libcxx]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      sanitizer: asan
+
+  seq-msan:
+    needs: [build-libcxx]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      sanitizer: msan
+
+  seq-ubsan:
+    needs: [build-libcxx]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      sanitizer: ubsan
+
+  # Parallel sanitizers: asan, ubsan
+  par-asan:
+    needs: [build-libcxx, build-hypre, build-metis]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      par: true
+      sanitizer: asan
+  par-ubsan:
+    needs: [build-libcxx, build-hypre, build-metis]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      par: true
+      sanitizer: ubsan

.gitignore

@@ -409,12 +409,14 @@ miniapps/tribol/contact-patch-test
 miniapps/diag-smoothers/abs-l1-jacobi
 miniapps/diag-smoothers/mg-abs-l1-jacobi
 
+miniapps/benchmarks/ceed-solver-bps/solver-bp
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/cunit_tests
-tests/unit/pcunit_tests
+tests/unit/gpu_unit_tests
+tests/unit/pgpu_unit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
@@ -429,6 +431,7 @@ tests/benchmarks/bench_ceed
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals
+tests/benchmarks/pbench_ceed
 
 # Test script output
 tests/scripts/*.err

CHANGELOG

@@ -39,6 +39,8 @@ GPU computing
   set. This is most often used for setting constant essential boundary
   conditions. A new function Vector::SetSubVectorHost has been added in cases
   where host execution is always needed (e.g. when the DOFs array is small).
+- Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
+  threads, assigning one task per thread.
 
 New and updated examples and miniapps
 -------------------------------------
@@ -48,13 +50,33 @@ New and updated examples and miniapps
   operators as smoothers.
   These miniapps can be found in `miniapps/diag-smoothers`.
 
-API changes:
+API changes
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
   mfem::future::tensor and mfem::future::dual.
 - API addition: in class `Operator`, added virtual functions: `AbsMult`, and
   `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.
 
+Miscellaneous
+-------------
+- Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
+  automatically select between CUDA or HIP.
+- The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
+  'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
+  CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
+  MFEM build configuration.
+- Added the option to enable GPU-aware MPI in MFEM using the environment
+  variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
+  variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
+- Added parallel Address Sanitizer, serial and parallel Undefined Behavior
+  Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
+- MFEM_PERF_* annotations: added options to enable GPU-stream- and
+  MPI-synchronizations at the start and at the end of annotation regions. These
+  synchronizations can be enabled or disabled (default) in code via the new
+  macros: MFEM_PERF_SYNC, MFEM_PERF_SYNC_STREAM, and MFEM_PERF_SYNC_MPI; the
+  environment variables with the same names can be set to 0/1 to control the
+  synchronization as well.
+
 
 Version 4.8, released on Apr 9, 2025
 ====================================

examples/ex26.cpp

@@ -78,6 +78,7 @@ private:
       opr.SetOperatorOwner(false);
 
       CGSolver* pcg = new CGSolver();
+      // pcg->iterative_mode = false; // the multigrid algorithm does this
       pcg->SetPrintLevel(-1);
       pcg->SetMaxIter(200);
       pcg->SetRelTol(sqrt(1e-4));
@@ -100,7 +101,8 @@ private:
       Vector diag(fespace.GetTrueVSize());
       bfs[level]->AssembleDiagonal(diag);
 
-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag, ess_tdof_list, 2);
+      Solver *smoother = new OperatorChebyshevSmoother(
+         *opr, diag, ess_tdof_list, 2);
       AddLevel(opr.Ptr(), smoother, true, true);
    }
 };

examples/ex26p.cpp

@@ -88,6 +88,7 @@ private:
       amg->SetPrintLevel(-1);
 
       CGSolver* pcg = new CGSolver(MPI_COMM_WORLD);
+      // pcg->iterative_mode = false; // the multigrid algorithm does this
       pcg->SetPrintLevel(-1);
       pcg->SetMaxIter(10);
       pcg->SetRelTol(sqrt(1e-4));

fem/CMakeLists.txt

@@ -247,6 +247,7 @@ set(HDRS
   nonlinearform_ext.hpp
   nonlininteg.hpp
   qfunction.hpp
+  qinterp/det.hpp
   qinterp/eval.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp

fem/batchitrans.cpp

@@ -515,6 +515,7 @@ struct InvTNewtonSolver<Geometry::SEGMENT, SDim, SType, max_team_x>
             phys_tol += pptr[idx + d * npts] * pptr[idx + d * npts];
          }
          phys_tol = fmax(phys_rtol * phys_rtol, phys_tol * phys_rtol * phys_rtol);
+         hit_bdr[0] = prev_hit_bdr[0] = false;
       }
       // for each iteration
       while (true)

fem/bilinearform_ext.cpp

@@ -255,6 +255,8 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
 
 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 {
+   MFEM_PERF_FUNCTION;
+
    if ( Device::Allows(Backend::CEED_MASK) ) { return; }
    ElementDofOrdering ordering = GetEVectorOrdering(*a->FESpace());
    elem_restrict = trial_fes->GetElementRestriction(ordering);
@@ -373,6 +375,8 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 
 void PABilinearFormExtension::Assemble()
 {
+   MFEM_PERF_FUNCTION;
+
    SetupRestrictionOperators(L2FaceValues::DoubleValued);
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
@@ -529,6 +533,8 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
                                            const bool useAbs) const
 {
+   MFEM_PERF_FUNCTION;
+
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
    const int iSz = integrators.Size();

fem/datacollection.cpp

@@ -764,9 +764,9 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 {
    cycle = 0;
 #ifdef MFEM_USE_ZLIB
-   compression = true; // if we have zlib, enable compression
-#else
-   compression = false; // otherwise, disable compression
+   // If we have zlib, enable compression. Otherwise, compression is disabled in
+   // the DataCollection base class constructor.
+   compression = true;
 #endif
 }
 
@@ -784,13 +784,8 @@ void ParaViewDataCollectionBase::SetCompressionLevel(int compression_level_)
 {
    MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9,
                "Compression level must be between -1 and 9 (inclusive).");
+   if (compression_level_ != 0) { SetCompression(true);}
    compression_level = compression_level_;
-   compression = compression_level_ != 0;
-}
-
-void ParaViewDataCollectionBase::SetCompression(bool compression_)
-{
-   compression = compression_;
 }
 
 int ParaViewDataCollectionBase::GetCompressionLevel() const
@@ -1174,7 +1169,14 @@ const char *ParaViewDataCollection::GetDataTypeString() const
 ParaViewHDFDataCollection::ParaViewHDFDataCollection(
    const std::string &collection_name, Mesh *mesh)
    : ParaViewDataCollectionBase(collection_name, mesh)
-{ }
+{
+   compression = true;
+}
+
+void ParaViewHDFDataCollection::SetCompression(bool compression_)
+{
+   compression = compression_;
+}
 
 void ParaViewHDFDataCollection::EnsureVTKHDF()
 {

fem/datacollection.hpp

@@ -537,13 +537,6 @@ public:
    /// Any nonzero compression level will enable compression.
    void SetCompressionLevel(int compression_level_);
 
-   /// @brief Enable or disable zlib compression.
-   ///
-   /// If the input is true, use the default zlib compression level (unless the
-   /// compression level has previously been set by calling
-   /// SetCompressionLevel()).
-   void SetCompression(bool compression_) override;
-
    /// @brief Sets whether or not to output the data as high-order elements
    /// (false by default).
    ///
@@ -633,6 +626,12 @@ public:
    ParaViewHDFDataCollection(const std::string& collection_name,
                              Mesh *mesh_ = nullptr);
 
+   /// @brief Enable or disable compression.
+   ///
+   /// The compression level can be set with SetCompressionLevel()). VTKHDF
+   /// compression does not require MFEM to be compiled with zlib support.
+   void SetCompression(bool compression_) override;
+
    /// Save the collection.
    void Save() override;
 

fem/dfem/parameterspace.hpp

@@ -101,10 +101,11 @@ public:
       // Setup DofToQuad information
       dtq.nqpt = (int)floor(std::pow(ir.GetNPoints(), 1.0 / mesh.Dimension()) + 0.5);
       dtq.ndof = dtq.nqpt;
+      dtq.mode = used_in_tensor_product ? DofToQuad::TENSOR : DofToQuad::FULL;
 
       // Calculate sizes
       const int num_qp = used_in_tensor_product ?
-                         std::pow(dtq.nqpt, mesh.Dimension()) :
+                         static_cast<int>(std::pow(dtq.nqpt, mesh.Dimension())) :
                          ir.GetNPoints();
 
       tsize = vdim * num_qp * mesh.GetNE();

fem/dfem/util.hpp

@@ -1702,12 +1702,13 @@ std::array<DofToQuadMap, N> load_dtq_mem(
    std::array<DofToQuadMap, N> f;
    for (std::size_t i = 0; i < N; i++)
    {
-      const auto [nqp_b, dim_b, ndof_b] = dtq[i].B.GetShape();
-      const auto B = Reshape(&dtq[i].B[0], nqp_b, dim_b, ndof_b);
-      auto mem_Bi = Reshape(reinterpret_cast<real_t *>(mem) + offset, nqp_b, dim_b,
-                            ndof_b);
       if (dtq[i].which_input != -1)
       {
+         const auto [nqp_b, dim_b, ndof_b] = dtq[i].B.GetShape();
+         const auto B = Reshape(&dtq[i].B[0], nqp_b, dim_b, ndof_b);
+         auto mem_Bi = Reshape(reinterpret_cast<real_t *>(mem) + offset, nqp_b, dim_b,
+                               ndof_b);
+
          MFEM_FOREACH_THREAD(q, x, nqp_b)
          {
             MFEM_FOREACH_THREAD(d, y, ndof_b)
@@ -2158,7 +2159,7 @@ template <
    std::size_t... Is>
 std::array<DofToQuadMap, N> create_dtq_maps_impl(
    field_operator_ts &fops,
-   std::vector<const DofToQuad*> dtqs,
+   std::vector<const DofToQuad*> &dtqs,
    const std::array<int, N> &field_map,
    std::index_sequence<Is...>)
 {
@@ -2243,7 +2244,7 @@ template <
    std::size_t num_fields>
 std::array<DofToQuadMap, num_fields> create_dtq_maps(
    field_operator_ts &fops,
-   std::vector<const DofToQuad*> dtqmaps,
+   std::vector<const DofToQuad*> &dtqmaps,
    const std::array<int, num_fields> &to_field_map)
 {
    return create_dtq_maps_impl<entity_t>(

fem/gridfunc.cpp

@@ -4334,7 +4334,7 @@ real_t LSZZErrorEstimator(BilinearFormIntegrator &blfi,  // input
          u.GetSubVector(udofs, ul);
          utrans.InvTransformPrimal(ul);
          Transf = ufes->GetElementTransformation(ielem);
-         FiniteElement *dummy = nullptr;
+         const auto *dummy = ufes->GetFE(ielem);
          blfi.ComputeElementFlux(*ufes->GetFE(ielem), *Transf, ul,
                                  *dummy, fl, with_coeff, ir);
 

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -1009,6 +1009,7 @@ inline void SmemPADiffusionApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_VERIFY(D1D <= Q1D, "THREAD_DIRECT requires D1D <= Q1D");
    mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -1038,11 +1039,11 @@ inline void SmemPADiffusionApply3D(const int NE,
       real_t (*QDD0)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+0);
       real_t (*QDD1)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+1);
       real_t (*QDD2)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
@@ -1050,9 +1051,9 @@ inline void SmemPADiffusionApply3D(const int NE,
       }
       if (MFEM_THREAD_ID(z) == 0)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                B[qx][dy] = b(qx,dy);
                G[qx][dy] = g(qx,dy);
@@ -1060,11 +1061,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                real_t u = 0.0, v = 0.0;
                MFEM_UNROLL(MD1)
@@ -1080,11 +1081,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MD1)
@@ -1101,11 +1102,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MD1)
@@ -1136,9 +1137,9 @@ inline void SmemPADiffusionApply3D(const int NE,
       MFEM_SYNC_THREAD;
       if (MFEM_THREAD_ID(z) == 0)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                Bt[dy][qx] = b(qx,dy);
                Gt[dy][qx] = g(qx,dy);
@@ -1146,11 +1147,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MQ1)
@@ -1167,11 +1168,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(Q1D)
@@ -1188,11 +1189,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MQ1)
@@ -1211,22 +1212,18 @@ inline void SmemPADiffusionApply3D(const int NE,
 
 } // namespace internal
 
-namespace
-{
-using ApplyKernelType = DiffusionIntegrator::ApplyKernelType;
-using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType;
-}
 
 template<int DIM, int T_D1D, int T_Q1D>
-ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel()
+DiffusionIntegrator::ApplyKernelType
+DiffusionIntegrator::ApplyPAKernels::Kernel()
 {
    if (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
    else if (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
    else { MFEM_ABORT(""); }
 }
 
-inline
-ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
+inline DiffusionIntegrator::ApplyKernelType
+DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
 {
    if (DIM == 2) { return internal::PADiffusionApply2D; }
    else if (DIM == 3) { return internal::PADiffusionApply3D; }
@@ -1234,14 +1231,15 @@ ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
 }
 
 template<int DIM, int D1D, int Q1D>
-DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel()
+DiffusionIntegrator::DiagonalKernelType
+DiffusionIntegrator::DiagonalPAKernels::Kernel()
 {
    if (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
    else if (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
    else { MFEM_ABORT(""); }
 }
 
-inline DiagonalKernelType
+inline DiffusionIntegrator::DiagonalKernelType
 DiffusionIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
 {
    if (DIM == 2) { return internal::PADiffusionDiagonal2D; }

fem/integ/bilininteg_diffusion_pa.cpp

@@ -39,6 +39,8 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 // PA Diffusion Apply kernel
 void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+   MFEM_PERF_FUNCTION;
+
    if (DeviceCanUseCeed())
    {
       ceedOp->AddMult(x, y);
@@ -88,6 +90,8 @@ void DiffusionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 
 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
+   MFEM_PERF_FUNCTION;
+
    const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
                          Device::GetDeviceMemoryType() : pa_mt;
    // Assuming the same element type

fem/integ/bilininteg_mass_kernels.hpp

@@ -1383,14 +1383,9 @@ inline void EAMassAssemble3D(const int NE,
 
 } // namespace internal
 
-namespace
-{
-using ApplyKernelType = MassIntegrator::ApplyKernelType;
-using DiagonalKernelType = MassIntegrator::DiagonalKernelType;
-}
 
 template<int DIM, int T_D1D, int T_Q1D>
-ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
+MassIntegrator::ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
 {
    if (DIM == 1) { return internal::PAMassApply1D; }
    else if (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
@@ -1398,8 +1393,8 @@ ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
    else { MFEM_ABORT(""); }
 }
 
-inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
-   int DIM, int, int)
+inline MassIntegrator::ApplyKernelType
+MassIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
 {
    if (DIM == 1) { return internal::PAMassApply1D; }
    else if (DIM == 2) { return internal::PAMassApply2D; }
@@ -1408,7 +1403,7 @@ inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
 }
 
 template<int DIM, int T_D1D, int T_Q1D>
-DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel()
+MassIntegrator::DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel()
 {
    if (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
    else if (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<T_D1D,T_Q1D>; }
@@ -1416,8 +1411,8 @@ DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel()
    else { MFEM_ABORT(""); }
 }
 
-inline DiagonalKernelType MassIntegrator::DiagonalPAKernels::Fallback(
-   int DIM, int, int)
+inline MassIntegrator::DiagonalKernelType
+MassIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
 {
    if (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
    else if (DIM == 2) { return internal::PAMassAssembleDiagonal2D; }

fem/integ/bilininteg_mass_pa.cpp

@@ -23,6 +23,8 @@ namespace mfem
 
 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
+   MFEM_PERF_FUNCTION;
+
    const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
                          Device::GetDeviceMemoryType() : pa_mt;
 
@@ -62,7 +64,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    const int NE = ne;
    const int Q1D = quad1D;
-   const int NQ = pow(Q1D, dim);
+   const int NQ = static_cast<int>(std::pow(Q1D, dim));
    const bool const_c = coeff.Size() == 1;
    const bool by_val = map_type == FiniteElement::VALUE;
    const auto W = Reshape(ir->GetWeights().Read(), NQ);
@@ -170,6 +172,8 @@ void MassIntegrator::AssembleDiagonalPA(Vector &diag)
 
 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+   MFEM_PERF_FUNCTION;
+
    if (DeviceCanUseCeed())
    {
       ceedOp->AddMult(x, y);

fem/linearform_ext.hpp

@@ -14,6 +14,7 @@
 
 #include "../general/array.hpp"
 #include "../linalg/vector.hpp"
+#include "fespace.hpp"
 
 namespace mfem
 {

fem/lininteg.hpp

@@ -673,7 +673,7 @@ public:
       int myid;
       MPI_Comm_rank(comm, &myid);
 
-      int seed = (seed_ > 0) ? seed_ + myid : (int)time(0) + myid;
+      int seed = (seed_ > 0) ? seed_ + myid : time(nullptr) + myid;
       SetSeed(seed);
    }
 #else

fem/lor/lor.cpp

@@ -365,6 +365,8 @@ FiniteElementSpace &LORBase::GetFESpace() const
 
 void LORBase::AssembleSystem(BilinearForm &a_ho, const Array<int> &ess_dofs)
 {
+   MFEM_PERF_FUNCTION;
+
    A.Clear();
    delete a;
    if (BatchedLORAssembly::FormIsSupported(a_ho))

fem/lor/lor_batched.cpp

@@ -360,6 +360,8 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
 
 void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
 {
+   MFEM_PERF_FUNCTION;
+
    const int nvdof = fes_ho.GetVSize();
 
    // If A contains an existing SparseMatrix, reuse it (and try to reuse its
@@ -417,6 +419,8 @@ static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim, int order)
 template <typename LOR_KERNEL>
 void BatchedLORAssembly::AssemblyKernel(BilinearForm &a)
 {
+   MFEM_PERF_FUNCTION;
+
    LOR_KERNEL kernel(a, fes_ho, X_vert, sparse_ij, sparse_mapping);
 
    const int dim = fes_ho.GetMesh()->Dimension();

fem/lor/lor_h1_impl.hpp

@@ -184,6 +184,8 @@ void BatchedLOR_H1::Assemble2D()
 template <int ORDER>
 void BatchedLOR_H1::Assemble3D()
 {
+   MFEM_PERF_FUNCTION;
+
    const int nel_ho = fes_ho.GetNE();
    static constexpr int nv = 8;
    static constexpr int dim = 3;

fem/multigrid.cpp

@@ -17,7 +17,10 @@ namespace mfem
 MultigridBase::MultigridBase()
    : cycleType(CycleType::VCYCLE), preSmoothingSteps(1), postSmoothingSteps(1),
      nrhs(0)
-{}
+{
+   coarse_solver = nullptr;
+   own_coarse_solver = false;
+}
 
 MultigridBase::MultigridBase(const Array<Operator*>& operators_,
                              const Array<Solver*>& smoothers_,
@@ -29,12 +32,18 @@ MultigridBase::MultigridBase(const Array<Operator*>& operators_,
 {
    operators_.Copy(operators);
    smoothers_.Copy(smoothers);
+   coarse_solver = nullptr;
    ownedOperators_.Copy(ownedOperators);
    ownedSmoothers_.Copy(ownedSmoothers);
+   own_coarse_solver = false;
 }
 
 MultigridBase::~MultigridBase()
 {
+   if (own_coarse_solver)
+   {
+      delete coarse_solver;
+   }
    for (int i = 0; i < operators.Size(); ++i)
    {
       if (ownedOperators[i])
@@ -56,16 +65,17 @@ void MultigridBase::InitVectors() const
    X.SetSize(M, nrhs);
    Y.SetSize(M, nrhs);
    R.SetSize(M, nrhs);
-   Z.SetSize(M, nrhs);
-   for (int i = 0; i < X.NumRows(); ++i)
+   for (int i = 0; i < M; ++i)
    {
       const int n = operators[i]->Height();
-      for (int j = 0; j < X.NumCols(); ++j)
+      for (int j = 0; j < nrhs; ++j)
       {
-         X(i, j) = new Vector(n);
-         Y(i, j) = new Vector(n);
+         if (i < M - 1)
+         {
+            X(i, j) = new Vector(n);
+            Y(i, j) = new Vector(n);
+         }
          R(i, j) = new Vector(n);
-         Z(i, j) = new Vector(n);
       }
    }
 }
@@ -76,10 +86,12 @@ void MultigridBase::EraseVectors() const
    {
       for (int j = 0; j < X.NumCols(); ++j)
       {
-         delete X(i, j);
-         delete Y(i, j);
+         if (i < X.NumRows() - 1)
+         {
+            delete X(i, j);
+            delete Y(i, j);
+         }
          delete R(i, j);
-         delete Z(i, j);
       }
    }
 }
@@ -95,6 +107,12 @@ void MultigridBase::AddLevel(Operator* op, Solver* smoother,
    ownedSmoothers.Append(ownSmoother);
 }
 
+void MultigridBase::AddCoarseSolver(Solver *c_solver, bool own_c_solver)
+{
+   coarse_solver = c_solver;
+   own_coarse_solver = own_c_solver;
+}
+
 void MultigridBase::SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
                                  int postSmoothingSteps_)
 {
@@ -105,10 +123,12 @@ void MultigridBase::SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
 
 void MultigridBase::Mult(const Vector& x, Vector& y) const
 {
-   Array<const Vector*> X_(1);
-   Array<Vector*> Y_(1);
-   X_[0] = &x;
-   Y_[0] = &y;
+   const Vector *x_array[1] = { &x };
+   Array<const Vector*> X_(x_array, 1); // no heap allocation
+
+   Vector *y_array[1] = { &y };
+   Array<Vector*> Y_(y_array, 1); // no heap allocation
+
    ArrayMult(X_, Y_);
 }
 
@@ -119,11 +139,6 @@ void MultigridBase::ArrayMult(const Array<const Vector*>& X_,
                "Multigrid solver does not have operators set!");
    MFEM_ASSERT(X_.Size() == Y_.Size(),
                "Number of columns mismatch in MultigridBase::Mult!");
-   if (iterative_mode)
-   {
-      MFEM_WARNING("Multigrid solver does not use iterative_mode and ignores "
-                   "the initial guess!");
-   }
 
    // Add capacity as necessary
    nrhs = X_.Size();
@@ -134,96 +149,159 @@ void MultigridBase::ArrayMult(const Array<const Vector*>& X_,
    for (int j = 0; j < nrhs; ++j)
    {
       MFEM_ASSERT(X_[j] && Y_[j], "Missing Vector in MultigridBase::Mult!");
-      *X(M - 1, j) = *X_[j];
-      *Y(M - 1, j) = 0.0;
-   }
-   Cycle(M - 1);
-   for (int j = 0; j < nrhs; ++j)
-   {
-      *Y_[j] = *Y(M - 1, j);
+      X(M - 1, j) = const_cast<Vector*>(X_[j]);
+      Y(M - 1, j) = Y_[j];
    }
+   const bool zero = !iterative_mode;
+   Cycle(M - 1, zero);
 }
 
 void MultigridBase::SmoothingStep(int level, bool zero, bool transpose) const
 {
    // y = y + S (x - A y) or y = y + S^T (x - A y)
+
+   // Note: 'zero' == true means that Y(level,*) are not initialized and we
+   // should assume that the input they typically provide to this call is zeros.
+
+   // We can't use the smoothers' iterative mode since we don't know if they
+   // actually support it, so we always turn the iterative mode off to properly
+   // use smoothers that do support it.
+   smoothers[level]->iterative_mode = false;
+
    if (zero)
    {
-      Array<Vector *> X_(X[level], nrhs), Y_(Y[level], nrhs);
-      GetSmootherAtLevel(level)->ArrayMult(X_, Y_);
+      MFEM_ASSERT(!transpose, "internal error!");
+      const Array<const Vector *> cX_((const Vector **)(X[level]), nrhs);
+      Array<Vector *> Y_(Y[level], nrhs);
+
+      GetSmootherAtLevel(level)->ArrayMult(cX_, Y_);
    }
    else
    {
-      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs),
-            Z_(Z[level], nrhs);
+      const Array<const Vector *> cY_((const Vector **)(Y[level]), nrhs),
+            cR_((const Vector **)(R[level]), nrhs);
+      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs);
+
+      GetOperatorAtLevel(level)->ArrayMult(cY_, R_);
       for (int j = 0; j < nrhs; ++j)
       {
-         *R_[j] = *X(level, j);
+         // *R_[j] = *X(level, j) - *R_[j]
+         subtract(*X(level, j), *R_[j], *R_[j]);
       }
-      GetOperatorAtLevel(level)->ArrayAddMult(Y_, R_, -1.0);
       if (transpose)
       {
-         GetSmootherAtLevel(level)->ArrayMultTranspose(R_, Z_);
+         GetSmootherAtLevel(level)->ArrayAddMultTranspose(cR_, Y_);
       }
       else
       {
-         GetSmootherAtLevel(level)->ArrayMult(R_, Z_);
-      }
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *Y_[j] += *Z_[j];
+         GetSmootherAtLevel(level)->ArrayAddMult(cR_, Y_);
       }
    }
 }
 
-void MultigridBase::Cycle(int level) const
+void MultigridBase::CoarseSolve(bool zero) const
 {
-   // Coarse solve
-   if (level == 0)
+   // See the comment about iterative mode in SmoothingStep()
+   coarse_solver->iterative_mode = false;
+
+   if (zero)
    {
-      SmoothingStep(0, true, false);
+      const Array<const Vector *> cX_((const Vector **)(X[0]), nrhs);
+      Array<Vector *> Y_(Y[0], nrhs);
+
+      coarse_solver->ArrayMult(cX_, Y_);
+   }
+   else
+   {
+      const Array<const Vector *> cY_((const Vector **)(Y[0]), nrhs),
+            cR_((const Vector **)(R[0]), nrhs);
+      Array<Vector *> Y_(Y[0], nrhs), R_(R[0], nrhs);
+
+      GetOperatorAtLevel(0)->ArrayMult(cY_, R_);
+      for (int j = 0; j < nrhs; ++j)
+      {
+         // *R_[j] = *X(0, j) - *R_[j]
+         subtract(*X(0, j), *R_[j], *R_[j]);
+      }
+      coarse_solver->ArrayAddMult(cR_, Y_);
+   }
+}
+
+void MultigridBase::Cycle(int level, bool zero) const
+{
+   // Note: 'zero' == true means that Y(level,*) are not initialized and we
+   // should assume that the input they typically provide to this call is zeros.
+
+   // Coarse solve
+   if (level == 0 && !coarse_solver)
+   {
+      SmoothingStep(0, zero, false);
       return;
    }
 
    // Pre-smooth
    for (int i = 0; i < preSmoothingSteps; ++i)
    {
-      SmoothingStep(level, (cycleType == CycleType::VCYCLE && i == 0), false);
+      SmoothingStep(level, zero && (i == 0), false);
+   }
+
+   // Coarse solve with 'coarse_solver'
+   if (level == 0)
+   {
+      CoarseSolve(preSmoothingSteps == 0 && zero);
+      goto mg_post_smooth;
    }
 
    // Compute residual and restrict
+   if (preSmoothingSteps == 0 && zero)
    {
-      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs),
-            X_(X[level - 1], nrhs);
+      const Array<const Vector *> cX_l((const Vector **)(X[level]), nrhs);
+      Array<Vector *> X_lm1(X[level - 1], nrhs);
+
+      GetProlongationAtLevel(level - 1)->ArrayMultTranspose(cX_l, X_lm1);
+   }
+   else
+   {
+      const Array<const Vector *> cY_((const Vector **)(Y[level]), nrhs),
+            cR_((const Vector **)(R[level]), nrhs);
+      Array<Vector *> R_(R[level], nrhs), X_(X[level - 1], nrhs);
+
+      GetOperatorAtLevel(level)->ArrayMult(cY_, R_);
       for (int j = 0; j < nrhs; ++j)
       {
-         *R_[j] = *X(level, j);
-      }
-      GetOperatorAtLevel(level)->ArrayAddMult(Y_, R_, -1.0);
-      GetProlongationAtLevel(level - 1)->ArrayMultTranspose(R_, X_);
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *Y(level - 1, j) = 0.0;
+         // *R_[j] = *X(level, j) - *R_[j]
+         subtract(*X(level, j), *R_[j], *R_[j]);
       }
+      GetProlongationAtLevel(level - 1)->ArrayMultTranspose(cR_, X_);
    }
 
    // Corrections
-   Cycle(level - 1);
+   Cycle(level - 1, true);
    if (cycleType == CycleType::WCYCLE)
    {
-      Cycle(level - 1);
+      // If the coarse solve at level 0 is "exact" solve, then we don't want to
+      // repeat it.
+      // To support multiple level 0 coarse-grid corrections, one can wrap that
+      // smoother in an SLI solver and use that instead.
+      if (level > 1) { Cycle(level - 1, false); }
    }
 
    // Prolongate and add
    {
-      Array<Vector *> Y_(Y[level - 1], nrhs), Z_(Z[level], nrhs);
-      GetProlongationAtLevel(level - 1)->ArrayMult(Y_, Z_);
-      for (int j = 0; j < nrhs; ++j)
+      const Array<const Vector *> cY_lm1((const Vector **)(Y[level - 1]), nrhs);
+      Array<Vector *> Y_l(Y[level], nrhs);
+
+      if (preSmoothingSteps == 0 && zero)
       {
-         *Y(level, j) += *Z_[j];
+         GetProlongationAtLevel(level - 1)->ArrayMult(cY_lm1, Y_l);
+      }
+      else
+      {
+         GetProlongationAtLevel(level - 1)->ArrayAddMult(cY_lm1, Y_l);
       }
    }
 
+mg_post_smooth:
    // Post-smooth
    for (int i = 0; i < postSmoothingSteps; ++i)
    {

fem/multigrid.hpp

@@ -36,12 +36,14 @@ protected:
    Array<Solver*> smoothers;
    Array<bool> ownedOperators;
    Array<bool> ownedSmoothers;
+   Solver *coarse_solver; /// can be NULL, see AddCoarseSolver()
+   bool own_coarse_solver;
 
    CycleType cycleType;
    int preSmoothingSteps;
    int postSmoothingSteps;
 
-   mutable Array2D<Vector*> X, Y, R, Z;
+   mutable Array2D<Vector*> X, Y, R;
    mutable int nrhs;
 
 public:
@@ -65,6 +67,16 @@ public:
    void AddLevel(Operator* op, Solver* smoother, bool ownOperator,
                  bool ownSmoother);
 
+   /// Adds a coarse solver for level 0 to work in tandem with the smoother
+   /** If this coarse solver is not given, the smoother at level 0 is used as
+       the coarse solver. When this coarse solver is given, the smoother at
+       level 0 is used similar to the smoothers at other levels. Thus, the
+       action at level 0 consists of:
+       - pre-smoothing steps with smoother 0,
+       - solve step with @a c_solver,
+       - post-smoothing steps with smoother 0. */
+   void AddCoarseSolver(Solver *c_solver, bool own_c_solver);
+
    /// Returns the number of levels
    int NumLevels() const { return operators.Size(); }
 
@@ -118,11 +130,14 @@ public:
 
 private:
    /// Application of a multigrid cycle at particular level
-   void Cycle(int level) const;
+   void Cycle(int level, bool zero) const;
 
    /// Application of a pre-/post-smoothing step at particular level
    void SmoothingStep(int level, bool zero, bool transpose) const;
 
+   /// Perform a coarse solve with 'coarse_solve' (must be non-NULL)
+   void CoarseSolve(bool zero) const;
+
    /// Allocate or destroy temporary storage
    void InitVectors() const;
    void EraseVectors() const;
@@ -202,6 +217,9 @@ public:
 
    /// Recover the solution of a linear system formed with FormFineLinearSystem()
    void RecoverFineFEMSolution(const Vector& X, const Vector& b, Vector& x);
+
+   const Array<int> &GetFineEssentialTrueDofs() const
+   { return *essentialTrueDofs.Last(); }
 };
 
 } // namespace mfem

fem/pbilinearform.cpp

@@ -124,6 +124,8 @@ void ParBilinearForm::pAllocMat()
 void ParBilinearForm::ParallelRAP(SparseMatrix &loc_A, OperatorHandle &A,
                                   bool steal_loc_A)
 {
+   MFEM_PERF_FUNCTION;
+
    ParFiniteElementSpace &pfespace = *ParFESpace();
 
    // Create a block diagonal parallel matrix

fem/pfespace.cpp

@@ -5259,7 +5259,7 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
       gc.GetNeighborLTDofTable(nbr_ltdof);
       const int nb_connections = nbr_ltdof.Size_of_connections();
       shr_ltdof.SetSize(nb_connections);
-      shr_ltdof.CopyFrom(nbr_ltdof.GetJ());
+      if (nb_connections > 0) { shr_ltdof.CopyFrom(nbr_ltdof.GetJ()); }
       shr_buf.SetSize(nb_connections);
       shr_buf.UseDevice(true);
       shr_buf_offsets = nbr_ltdof.GetIMemory();
@@ -5288,7 +5288,7 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
       gc.GetNeighborLDofTable(nbr_ldof);
       const int nb_connections = nbr_ldof.Size_of_connections();
       ext_ldof.SetSize(nb_connections);
-      ext_ldof.CopyFrom(nbr_ldof.GetJ());
+      if (nb_connections > 0) { ext_ldof.CopyFrom(nbr_ldof.GetJ()); }
       ext_ldof.GetMemory().UseDevice(true);
       ext_buf.SetSize(nb_connections);
       ext_buf.UseDevice(true);
@@ -5309,6 +5309,18 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
       if (recv_size > 0) { req_counter++; }
    }
    requests = new MPI_Request[req_counter];
+   if (mpi_gpu_aware)
+   {
+#if defined(MFEM_USE_HIP)
+      MFEM_GPU_CHECK(
+         hipEventCreateWithFlags(&gpu_event, hipEventDisableTiming));
+#elif defined(MFEM_USE_CUDA)
+      MFEM_GPU_CHECK(
+         cudaEventCreateWithFlags(&gpu_event, cudaEventDisableTiming));
+#else
+      MFEM_ABORT("not implemented");
+#endif
+   }
 }
 
 DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
@@ -5322,16 +5334,27 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
 }
 
 static void ExtractSubVector(const Array<int> &indices,
-                             const Vector &vin, Vector &vout)
+                             const Vector &vin, Vector &vout,
+                             real_t a, real_t b)
 {
    MFEM_ASSERT(indices.Size() == vout.Size(), "incompatible sizes!");
-   auto y = vout.Write();
+   auto y = (a == 0) ? vout.Write() : vout.ReadWrite();
    const auto x = vin.Read();
    const auto I = indices.Read();
-   mfem::forall(indices.Size(), [=] MFEM_HOST_DEVICE (int i)
+   if (a == 0)
    {
-      y[i] = x[I[i]];
-   }); // indices can be repeated
+      mfem::forall(indices.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         y[i] = b*x[I[i]];
+      }); // indices can be repeated
+   }
+   else
+   {
+      mfem::forall(indices.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         y[i] = a*y[i] + b*x[I[i]];
+      }); // indices can be repeated
+   }
 }
 
 void DeviceConformingProlongationOperator::BcastBeginCopy(
@@ -5339,10 +5362,7 @@ void DeviceConformingProlongationOperator::BcastBeginCopy(
 {
    // shr_buf[i] = src[shr_ltdof[i]]
    if (shr_ltdof.Size() == 0) { return; }
-   ExtractSubVector(shr_ltdof, x, shr_buf);
-   // If the above kernel is executed asynchronously, we should wait for it to
-   // complete
-   if (mpi_gpu_aware) { MFEM_STREAM_SYNC; }
+   ExtractSubVector(shr_ltdof, x, shr_buf, 0, 1);
 }
 
 static void SetSubVector(const Array<int> &indices,
@@ -5379,7 +5399,7 @@ void DeviceConformingProlongationOperator::Mult(const Vector &x,
                                                 Vector &y) const
 {
    const GroupTopology &gtopo = gc.GetGroupTopology();
-   int req_counter = 0;
+   int req_counter = 0, num_recv_req = 0;
    // Make sure 'y' is marked as valid on device and for use on device.
    // This ensures that there is no unnecessary host to device copy when the
    // input 'y' is valid on host (in 'y.SetSubVector(ext_ldof, 0.0)' when local
@@ -5389,42 +5409,89 @@ void DeviceConformingProlongationOperator::Mult(const Vector &x,
    {
       // done on device since we've marked ext_ldof for use on device:
       y.SetSubVector(ext_ldof, 0.0);
+      BcastLocalCopy(x, y);
+      return;
    }
-   else
+
+   BcastBeginCopy(x); // copy to 'shr_buf'
+   if (mpi_gpu_aware && shr_ltdof.Size() != 0)
    {
-      BcastBeginCopy(x); // copy to 'shr_buf'
+      /* record a stream event to wait for later */
+#if defined(MFEM_USE_HIP)
+      MFEM_GPU_CHECK(hipEventRecord(gpu_event, 0));
+#elif defined(MFEM_USE_CUDA)
+      MFEM_GPU_CHECK(cudaEventRecord(gpu_event, 0));
+#endif
+   }
+   BcastLocalCopy(x, y);
+   // Queue all receive communications
+   if (ext_ldof.Size() != 0) // ext_ldof.Size() == ext_buf.Size()
+   {
+      auto recv_buf = mpi_gpu_aware ? ext_buf.Write() : ext_buf.HostWrite();
+      for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+      {
+         const int recv_offset = ext_buf_offsets[nbr];
+         const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
+         if (recv_size > 0)
+         {
+            MPI_Irecv(recv_buf + recv_offset, recv_size,
+                      MPITypeMap<real_t>::mpi_type,
+                      gtopo.GetNeighborRank(nbr), 41822,
+                      gtopo.GetComm(), &requests[req_counter++]);
+         }
+      }
+      num_recv_req = req_counter;
+   }
+   // Queue all send communications
+   if (shr_ltdof.Size() != 0) // shr_ltdof.Size() == shr_buf.Size()
+   {
+      // The BcastBeginCopy kernel is executed asynchronously, we should wait
+      // for it to complete:
+      // - when mpi_gpu_aware == false, this is done implicily when we call
+      //   shr_buf.HostRead()
+      // - when mpi_gpu_aware == true, we need to wait for BcastBeginCopy to
+      //   complete by waiting for gpu_event.
+      if (mpi_gpu_aware)
+      {
+         /* wait for the stream event recorded above */
+#if defined(MFEM_USE_HIP)
+         MFEM_GPU_CHECK(hipEventSynchronize(gpu_event));
+#elif defined(MFEM_USE_CUDA)
+         MFEM_GPU_CHECK(cudaEventSynchronize(gpu_event));
+#endif
+      }
+      auto send_buf = mpi_gpu_aware ? shr_buf.Read() : shr_buf.HostRead();
       for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
       {
          const int send_offset = shr_buf_offsets[nbr];
          const int send_size = shr_buf_offsets[nbr+1] - send_offset;
          if (send_size > 0)
          {
-            auto send_buf = mpi_gpu_aware ? shr_buf.Read() : shr_buf.HostRead();
-            MPI_Isend(send_buf + send_offset, send_size, MPITypeMap<real_t>::mpi_type,
-                      gtopo.GetNeighborRank(nbr), 41822,
-                      gtopo.GetComm(), &requests[req_counter++]);
-         }
-         const int recv_offset = ext_buf_offsets[nbr];
-         const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-         if (recv_size > 0)
-         {
-            auto recv_buf = mpi_gpu_aware ? ext_buf.Write() : ext_buf.HostWrite();
-            MPI_Irecv(recv_buf + recv_offset, recv_size, MPITypeMap<real_t>::mpi_type,
+            MPI_Isend(send_buf + send_offset, send_size,
+                      MPITypeMap<real_t>::mpi_type,
                       gtopo.GetNeighborRank(nbr), 41822,
                       gtopo.GetComm(), &requests[req_counter++]);
          }
       }
    }
-   BcastLocalCopy(x, y);
-   if (!local)
-   {
-      MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-      BcastEndCopy(y); // copy from 'ext_buf'
-   }
+   // Wait for all receive requests
+   MPI_Waitall(num_recv_req, requests, MPI_STATUSES_IGNORE);
+   BcastEndCopy(y); // copy from 'ext_buf'
+   // Wait for all send requests
+   MPI_Waitall(req_counter - num_recv_req, requests + num_recv_req,
+               MPI_STATUSES_IGNORE);
 }
 
 DeviceConformingProlongationOperator::~DeviceConformingProlongationOperator()
 {
+   if (mpi_gpu_aware)
+   {
+#if defined(MFEM_USE_HIP)
+      MFEM_GPU_CHECK(hipEventDestroy(gpu_event));
+#elif defined(MFEM_USE_CUDA)
+      MFEM_GPU_CHECK(cudaEventDestroy(gpu_event));
+#endif
+   }
    delete [] requests;
    ext_buf_offsets.Delete();
    shr_buf_offsets.Delete();
@@ -5435,25 +5502,23 @@ void DeviceConformingProlongationOperator::ReduceBeginCopy(
 {
    // ext_buf[i] = src[ext_ldof[i]]
    if (ext_ldof.Size() == 0) { return; }
-   ExtractSubVector(ext_ldof, x, ext_buf);
-   // If the above kernel is executed asynchronously, we should wait for it to
-   // complete
-   if (mpi_gpu_aware) { MFEM_STREAM_SYNC; }
+   ExtractSubVector(ext_ldof, x, ext_buf, 0, 1);
 }
 
 void DeviceConformingProlongationOperator::ReduceLocalCopy(
-   const Vector &x, Vector &y) const
+   const Vector &x, Vector &y, real_t a, real_t b) const
 {
    // dst[i] = src[ltdof_ldof[i]]
    if (ltdof_ldof.Size() == 0) { return; }
-   ExtractSubVector(ltdof_ldof, x, y);
+   ExtractSubVector(ltdof_ldof, x, y, a, b);
 }
 
 static void AddSubVector(const Array<int> &unique_dst_indices,
                          const Array<int> &unique_to_src_offsets,
                          const Array<int> &unique_to_src_indices,
                          const Vector &src,
-                         Vector &dst)
+                         Vector &dst,
+                         real_t b)
 {
    auto y = dst.ReadWrite();
    const auto x = src.Read();
@@ -5463,56 +5528,94 @@ static void AddSubVector(const Array<int> &unique_dst_indices,
    mfem::forall(unique_dst_indices.Size(), [=] MFEM_HOST_DEVICE (int i)
    {
       const int dst_idx = DST_I[i];
-      real_t sum = y[dst_idx];
+      real_t sum = 0;
       const int end = SRC_O[i+1];
       for (int j = SRC_O[i]; j != end; ++j) { sum += x[SRC_I[j]]; }
-      y[dst_idx] = sum;
+      y[dst_idx] += b*sum;
    });
 }
 
-void DeviceConformingProlongationOperator::ReduceEndAssemble(Vector &y) const
+void DeviceConformingProlongationOperator::ReduceEndAssemble(
+   Vector &y, real_t b) const
 {
    // dst[shr_ltdof[i]] += shr_buf[i]
    if (unq_ltdof.Size() == 0) { return; }
-   AddSubVector(unq_ltdof, unq_shr_i, unq_shr_j, shr_buf, y);
+   AddSubVector(unq_ltdof, unq_shr_i, unq_shr_j, shr_buf, y, b);
 }
 
-void DeviceConformingProlongationOperator::MultTranspose(const Vector &x,
-                                                         Vector &y) const
+void DeviceConformingProlongationOperator::ApplyTranspose(
+   const Vector &x, Vector &y, real_t a, real_t b) const
 {
    const GroupTopology &gtopo = gc.GetGroupTopology();
-   int req_counter = 0;
-   if (!local)
+   int req_counter = 0, num_recv_req = 0;
+
+   if (local)
    {
-      ReduceBeginCopy(x); // copy to 'ext_buf'
+      ReduceLocalCopy(x, y, a, b);
+      return;
+   }
+
+   ReduceBeginCopy(x); // copy to 'ext_buf'
+   if (mpi_gpu_aware && ext_ldof.Size() != 0)
+   {
+      /* record a stream event to wait for later */
+#if defined(MFEM_USE_HIP)
+      MFEM_GPU_CHECK(hipEventRecord(gpu_event, 0));
+#elif defined(MFEM_USE_CUDA)
+      MFEM_GPU_CHECK(cudaEventRecord(gpu_event, 0));
+#endif
+   }
+   ReduceLocalCopy(x, y, a, b);
+   // Queue all receive communications
+   if (unq_ltdof.Size() != 0)
+   {
+      auto recv_buf = mpi_gpu_aware ? shr_buf.Write() : shr_buf.HostWrite();
+      for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
+      {
+         const int recv_offset = shr_buf_offsets[nbr];
+         const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
+         if (recv_size > 0)
+         {
+            MPI_Irecv(recv_buf + recv_offset, recv_size,
+                      MPITypeMap<real_t>::mpi_type,
+                      gtopo.GetNeighborRank(nbr), 41823,
+                      gtopo.GetComm(), &requests[req_counter++]);
+         }
+      }
+      num_recv_req = req_counter;
+   }
+   // Queue all send communications
+   if (ext_ldof.Size() != 0)
+   {
+      if (mpi_gpu_aware)
+      {
+         /* wait for the stream event recorded above */
+#if defined(MFEM_USE_HIP)
+         MFEM_GPU_CHECK(hipEventSynchronize(gpu_event));
+#elif defined(MFEM_USE_CUDA)
+         MFEM_GPU_CHECK(cudaEventSynchronize(gpu_event));
+#endif
+      }
+      auto send_buf = mpi_gpu_aware ? ext_buf.Read() : ext_buf.HostRead();
       for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
       {
          const int send_offset = ext_buf_offsets[nbr];
          const int send_size = ext_buf_offsets[nbr+1] - send_offset;
          if (send_size > 0)
          {
-            auto send_buf = mpi_gpu_aware ? ext_buf.Read() : ext_buf.HostRead();
-            MPI_Isend(send_buf + send_offset, send_size, MPITypeMap<real_t>::mpi_type,
-                      gtopo.GetNeighborRank(nbr), 41823,
-                      gtopo.GetComm(), &requests[req_counter++]);
-         }
-         const int recv_offset = shr_buf_offsets[nbr];
-         const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
-         if (recv_size > 0)
-         {
-            auto recv_buf = mpi_gpu_aware ? shr_buf.Write() : shr_buf.HostWrite();
-            MPI_Irecv(recv_buf + recv_offset, recv_size, MPITypeMap<real_t>::mpi_type,
+            MPI_Isend(send_buf + send_offset, send_size,
+                      MPITypeMap<real_t>::mpi_type,
                       gtopo.GetNeighborRank(nbr), 41823,
                       gtopo.GetComm(), &requests[req_counter++]);
          }
       }
    }
-   ReduceLocalCopy(x, y);
-   if (!local)
-   {
-      MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-      ReduceEndAssemble(y); // assemble from 'shr_buf'
-   }
+   // Wait for all receive requests
+   MPI_Waitall(num_recv_req, requests, MPI_STATUSES_IGNORE);
+   ReduceEndAssemble(y, b); // assemble from 'shr_buf'
+   // Wait for all send requests
+   MPI_Waitall(req_counter - num_recv_req, requests + num_recv_req,
+               MPI_STATUSES_IGNORE);
 }
 
 } // namespace mfem

fem/pfespace.hpp

@@ -586,6 +586,7 @@ public:
    { MultTranspose(x,y); }
 };
 
+
 /// Auxiliary device class used by ParFiniteElementSpace.
 class DeviceConformingProlongationOperator: public
    ConformingProlongationOperator
@@ -598,6 +599,11 @@ protected:
    Array<int> ltdof_ldof, unq_ltdof;
    Array<int> unq_shr_i, unq_shr_j;
    MPI_Request *requests;
+#if defined(MFEM_USE_HIP)
+   hipEvent_t gpu_event;
+#elif defined(MFEM_USE_CUDA)
+   cudaEvent_t gpu_event;
+#endif
 
    // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
    //         shr_buf[i] = src[shr_ltdof[i]]
@@ -617,11 +623,12 @@ protected:
 
    // Kernel: copy owned ldofs from 'src' to ltdofs in 'dst'.
    //         dst[i] = src[ltdof_ldof[i]]
-   void ReduceLocalCopy(const Vector &src, Vector &dst) const;
+   void ReduceLocalCopy(const Vector &src, Vector &dst,
+                        real_t a, real_t b) const;
 
    // Kernel: assemble dofs from 'shr_buf' into to 'dst' - after recv.
    //         dst[shr_ltdof[i]] += shr_buf[i]
-   void ReduceEndAssemble(Vector &dst) const;
+   void ReduceEndAssemble(Vector &dst, real_t b) const;
 
 public:
    DeviceConformingProlongationOperator(
@@ -632,12 +639,20 @@ public:
 
    virtual ~DeviceConformingProlongationOperator();
 
+   void ApplyTranspose(const Vector &x, Vector &y,
+                       real_t a, real_t b) const;
+
    void Mult(const Vector &x, Vector &y) const override;
 
    void AbsMult(const Vector &x, Vector &y) const override
    { Mult(x,y); }
 
-   void MultTranspose(const Vector &x, Vector &y) const override;
+   void MultTranspose(const Vector &x, Vector &y) const override
+   { ApplyTranspose(x, y, 0, 1); }
+
+   void AddMultTranspose(const Vector &x, Vector &y,
+                         real_t a = 1) const override
+   { ApplyTranspose(x, y, 1, a); }
 
    void AbsMultTranspose(const Vector &x, Vector &y) const override
    { MultTranspose(x,y); }

fem/qinterp/det.cpp

@@ -9,278 +9,16 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../quadinterpolator.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../fem/kernels.hpp"
-#include "../../linalg/kernels.hpp"
-
-using namespace mfem;
+#include "det.hpp"
 
 namespace mfem
 {
 
 namespace internal
 {
-
 namespace quadrature_interpolator
 {
 
-static void Det1D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d,
-                  const int q1d,
-                  Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(b);
-   MFEM_CONTRACT_VAR(d_buff);
-   const auto G = Reshape(g, q1d, d1d);
-   const auto X = Reshape(x, d1d, NE);
-
-   auto Y = Reshape(y, q1d, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int q = 0; q < q1d; q++)
-      {
-         real_t u = 0.0;
-         for (int d = 0; d < d1d; d++)
-         {
-            u += G(q, d) * X(d, e);
-         }
-         Y(q, e) = u;
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void Det2D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d = 0,
-                  const int q1d = 0,
-                  Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(d_buff);
-   static constexpr int SDIM = 2;
-   static constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, NE);
-
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t XY[SDIM][NBZ][MD1*MD1];
-      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
-      MFEM_SHARED real_t QQ[2*SDIM][NBZ][MQ1*MQ1];
-
-      kernels::internal::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::internal::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::internal::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            real_t J[4];
-            kernels::internal::PullGrad<MQ1,NBZ>(Q1D,qx,qy,QQ,J);
-            Y(qx,qy,e) = kernels::Det<2>(J);
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void Det2DSurface(const int NE,
-                         const real_t *b,
-                         const real_t *g,
-                         const real_t *x,
-                         real_t *y,
-                         const int d1d = 0,
-                         const int q1d = 0,
-                         Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(d_buff);
-
-   static constexpr int SDIM = 3;
-   static constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, NE);
-
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t XYZ[SDIM][NBZ][MD1*MD1];
-      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
-
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      // Load XYZ components
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int d = 0; d < SDIM; ++d)
-            {
-               XYZ[d][tidz][dx + dy*D1D] = X(dx,dy,d,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      ConstDeviceMatrix B_mat(BG[0], D1D, Q1D);
-      ConstDeviceMatrix G_mat(BG[1], D1D, Q1D);
-
-      // x contraction
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            for (int d = 0; d < SDIM; ++d)
-            {
-               real_t u = 0.0;
-               real_t v = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t xval = XYZ[d][tidz][dx + dy*D1D];
-                  u += xval * G_mat(dx,qx);
-                  v += xval * B_mat(dx,qx);
-               }
-               DQ[d][tidz][dy + qx*D1D] = u;
-               DQ[3 + d][tidz][dy + qx*D1D] = v;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // y contraction and determinant computation
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            real_t J_[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
-            for (int d = 0; d < SDIM; ++d)
-            {
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  J_[d] += DQ[d][tidz][dy + qx*D1D] * B_mat(dy,qy);
-                  J_[3 + d] += DQ[3 + d][tidz][dy + qx*D1D] * G_mat(dy,qy);
-               }
-            }
-            DeviceTensor<2> J(J_, 3, 2);
-            const real_t E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
-            const real_t F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
-            const real_t G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
-            Y(qx,qy,e) = std::sqrt(E*G - F*F);
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, bool SMEM = true>
-static void Det3D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d = 0,
-                  const int q1d = 0,
-                  Vector *d_buff = nullptr) // used only with SMEM = false
-{
-   constexpr int DIM = 3;
-   static constexpr int GRID = SMEM ? 0 : 128;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
-
-   real_t *GM = nullptr;
-   if (!SMEM)
-   {
-      const DeviceDofQuadLimits &limits = DeviceDofQuadLimits::Get();
-      const int max_q1d = T_Q1D ? T_Q1D : limits.MAX_Q1D;
-      const int max_d1d = T_D1D ? T_D1D : limits.MAX_D1D;
-      const int max_qd = std::max(max_q1d, max_d1d);
-      const int mem_size = max_qd * max_qd * max_qd * 9;
-      d_buff->SetSize(2*mem_size*GRID);
-      GM = d_buff->Write();
-   }
-
-   mfem::forall_3D_grid(NE, Q1D, Q1D, Q1D, GRID, [=] MFEM_HOST_DEVICE (int e)
-   {
-      static constexpr int MQ1 = T_Q1D ? T_Q1D :
-                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_Q1D);
-      static constexpr int MD1 = T_D1D ? T_D1D :
-                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_D1D);
-      static constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
-      static constexpr int MSZ = MDQ * MDQ * MDQ * 9;
-
-      const int bid = MFEM_BLOCK_ID(x);
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t SM0[SMEM?MSZ:1];
-      MFEM_SHARED real_t SM1[SMEM?MSZ:1];
-      real_t *lm0 = SMEM ? SM0 : GM + MSZ*bid;
-      real_t *lm1 = SMEM ? SM1 : GM + MSZ*(GRID+bid);
-      real_t (*DDD)[MD1*MD1*MD1] = (real_t (*)[MD1*MD1*MD1]) (lm0);
-      real_t (*DDQ)[MD1*MD1*MQ1] = (real_t (*)[MD1*MD1*MQ1]) (lm1);
-      real_t (*DQQ)[MD1*MQ1*MQ1] = (real_t (*)[MD1*MQ1*MQ1]) (lm0);
-      real_t (*QQQ)[MQ1*MQ1*MQ1] = (real_t (*)[MQ1*MQ1*MQ1]) (lm1);
-
-      kernels::internal::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::internal::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::internal::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::internal::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t J[9];
-               kernels::internal::PullGrad<MQ1>(Q1D, qx,qy,qz, QQQ, J);
-               Y(qx,qy,qz,e) = kernels::Det<3>(J);
-            }
-         }
-      }
-   });
-}
-
 void InitDetKernels()
 {
    using k = QuadratureInterpolator::DetKernels;
@@ -302,27 +40,12 @@ void InitDetKernels()
 }
 
 } // namespace quadrature_interpolator
-
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-namespace
-{
-using DetKernel = QuadratureInterpolator::DetKernelType;
-}
-
-template<int DIM, int SDIM, int D1D, int Q1D>
-DetKernel QuadratureInterpolator::DetKernels::Kernel()
-{
-   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
-   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
-   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
-}
-
-DetKernel QuadratureInterpolator::DetKernels::Fallback(
+QuadratureInterpolator::DetKernelType
+QuadratureInterpolator::DetKernels::Fallback(
    int DIM, int SDIM, int D1D, int Q1D)
 {
    if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }

fem/qinterp/det.hpp

@@ -0,0 +1,304 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_QUADINTERP_DET_HPP
+#define MFEM_QUADINTERP_DET_HPP
+
+#include "../quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../fem/kernels.hpp"
+#include "../../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+inline void Det1D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d,
+                  const int q1d,
+                  Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(b);
+   MFEM_CONTRACT_VAR(d_buff);
+   const auto G = Reshape(g, q1d, d1d);
+   const auto X = Reshape(x, d1d, NE);
+
+   auto Y = Reshape(y, q1d, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int q = 0; q < q1d; q++)
+      {
+         real_t u = 0.0;
+         for (int d = 0; d < d1d; d++)
+         {
+            u += G(q, d) * X(d, e);
+         }
+         Y(q, e) = u;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void Det2D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d = 0,
+                  const int q1d = 0,
+                  Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(d_buff);
+   static constexpr int SDIM = 2;
+   static constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, NE);
+
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t XY[SDIM][NBZ][MD1*MD1];
+      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
+      MFEM_SHARED real_t QQ[2*SDIM][NBZ][MQ1*MQ1];
+
+      kernels::internal::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::internal::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::internal::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            real_t J[4];
+            kernels::internal::PullGrad<MQ1,NBZ>(Q1D,qx,qy,QQ,J);
+            Y(qx,qy,e) = kernels::Det<2>(J);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void Det2DSurface(const int NE,
+                         const real_t *b,
+                         const real_t *g,
+                         const real_t *x,
+                         real_t *y,
+                         const int d1d = 0,
+                         const int q1d = 0,
+                         Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(d_buff);
+
+   static constexpr int SDIM = 3;
+   static constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, NE);
+
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t XYZ[SDIM][NBZ][MD1*MD1];
+      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
+
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      // Load XYZ components
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int d = 0; d < SDIM; ++d)
+            {
+               XYZ[d][tidz][dx + dy*D1D] = X(dx,dy,d,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      ConstDeviceMatrix B_mat(BG[0], D1D, Q1D);
+      ConstDeviceMatrix G_mat(BG[1], D1D, Q1D);
+
+      // x contraction
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            for (int d = 0; d < SDIM; ++d)
+            {
+               real_t u = 0.0;
+               real_t v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t xval = XYZ[d][tidz][dx + dy*D1D];
+                  u += xval * G_mat(dx,qx);
+                  v += xval * B_mat(dx,qx);
+               }
+               DQ[d][tidz][dy + qx*D1D] = u;
+               DQ[3 + d][tidz][dy + qx*D1D] = v;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // y contraction and determinant computation
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            real_t J_[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
+            for (int d = 0; d < SDIM; ++d)
+            {
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  J_[d] += DQ[d][tidz][dy + qx*D1D] * B_mat(dy,qy);
+                  J_[3 + d] += DQ[3 + d][tidz][dy + qx*D1D] * G_mat(dy,qy);
+               }
+            }
+            DeviceTensor<2> J(J_, 3, 2);
+            const real_t E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
+            const real_t F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
+            const real_t G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
+            Y(qx,qy,e) = std::sqrt(E*G - F*F);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, bool SMEM = true>
+inline void Det3D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d = 0,
+                  const int q1d = 0,
+                  Vector *d_buff = nullptr) // used only with SMEM = false
+{
+   constexpr int DIM = 3;
+   static constexpr int GRID = SMEM ? 0 : 128;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
+
+   real_t *GM = nullptr;
+   if (!SMEM)
+   {
+      const DeviceDofQuadLimits &limits = DeviceDofQuadLimits::Get();
+      const int max_q1d = T_Q1D ? T_Q1D : limits.MAX_Q1D;
+      const int max_d1d = T_D1D ? T_D1D : limits.MAX_D1D;
+      const int max_qd = std::max(max_q1d, max_d1d);
+      const int mem_size = max_qd * max_qd * max_qd * 9;
+      d_buff->SetSize(2*mem_size*GRID);
+      GM = d_buff->Write();
+   }
+
+   mfem::forall_3D_grid(NE, Q1D, Q1D, Q1D, GRID, [=] MFEM_HOST_DEVICE (int e)
+   {
+      static constexpr int MQ1 = T_Q1D ? T_Q1D :
+                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_Q1D);
+      static constexpr int MD1 = T_D1D ? T_D1D :
+                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_D1D);
+      static constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
+      static constexpr int MSZ = MDQ * MDQ * MDQ * 9;
+
+      const int bid = MFEM_BLOCK_ID(x);
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t SM0[SMEM?MSZ:1];
+      MFEM_SHARED real_t SM1[SMEM?MSZ:1];
+      real_t *lm0 = SMEM ? SM0 : GM + MSZ*bid;
+      real_t *lm1 = SMEM ? SM1 : GM + MSZ*(GRID+bid);
+      real_t (*DDD)[MD1*MD1*MD1] = (real_t (*)[MD1*MD1*MD1]) (lm0);
+      real_t (*DDQ)[MD1*MD1*MQ1] = (real_t (*)[MD1*MD1*MQ1]) (lm1);
+      real_t (*DQQ)[MD1*MQ1*MQ1] = (real_t (*)[MD1*MQ1*MQ1]) (lm0);
+      real_t (*QQQ)[MQ1*MQ1*MQ1] = (real_t (*)[MQ1*MQ1*MQ1]) (lm1);
+
+      kernels::internal::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::internal::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::internal::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::internal::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t J[9];
+               kernels::internal::PullGrad<MQ1>(Q1D, qx,qy,qz, QQQ, J);
+               Y(qx,qy,qz,e) = kernels::Det<3>(J);
+            }
+         }
+      }
+   });
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+
+/// @cond Suppress_Doxygen_warnings
+
+template<int DIM, int SDIM, int D1D, int Q1D>
+QuadratureInterpolator::DetKernelType
+QuadratureInterpolator::DetKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
+   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
+   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+}
+
+/// @endcond
+
+} // namespace mfem
+
+#endif // MFEM_QUADINTERP_DET_HPP

fem/quadinterpolator.cpp

@@ -503,6 +503,8 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
                                   Vector &q_der,
                                   Vector &q_det) const
 {
+   MFEM_PERF_FUNCTION;
+
    using namespace internal::quadrature_interpolator;
 
    const int ne = fespace->GetNE();

fem/tmop.cpp

@@ -5122,33 +5122,32 @@ real_t TMOP_Integrator::GetSurfaceFittingWeight()
 
 void TMOP_Integrator::EnableNormalization(const GridFunction &x)
 {
-   ComputeNormalizationEnergies(x, metric_normal, lim_normal, surf_fit_normal);
+   ComputeNormalizationEnergies(x, metric_normal, lim_normal);
    metric_normal = 1.0 / metric_normal;
    lim_normal = 1.0 / lim_normal;
-   //if (surf_fit_gf) { surf_fit_normal = 1.0 / surf_fit_normal; }
    if (surf_fit_gf || surf_fit_pos) { surf_fit_normal = lim_normal; }
 }
 
 #ifdef MFEM_USE_MPI
 void TMOP_Integrator::ParEnableNormalization(const ParGridFunction &x)
 {
-   real_t loc[3];
-   ComputeNormalizationEnergies(x, loc[0], loc[1], loc[2]);
-   real_t rdc[3];
-   MPI_Allreduce(loc, rdc, 3, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+   real_t loc[2];
+   ComputeNormalizationEnergies(x, loc[0], loc[1]);
+   real_t rdc[2];
+   MPI_Allreduce(loc, rdc, 2, MPITypeMap<real_t>::mpi_type, MPI_SUM,
                  x.ParFESpace()->GetComm());
    metric_normal = 1.0 / rdc[0];
    lim_normal    = 1.0 / rdc[1];
-   // if (surf_fit_gf) { surf_fit_normal = 1.0 / rdc[2]; }
    if (surf_fit_gf || surf_fit_pos) { surf_fit_normal = lim_normal; }
 }
 #endif
 
 void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
                                                    real_t &metric_energy,
-                                                   real_t &lim_energy,
-                                                   real_t &surf_fit_gf_energy)
+                                                   real_t &lim_energy)
 {
+   metric_energy = 0.0;
+   lim_energy = 0.0;
    if (PA.enabled)
    {
       MFEM_VERIFY(PA.E.Size() > 0, "Must be called after AssemblePA!");
@@ -5191,9 +5190,6 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
    Jpr.SetSize(dim);
    Jpt.SetSize(dim);
 
-   metric_energy = 0.0;
-   lim_energy = 0.0;
-   surf_fit_gf_energy = 0.0;
    for (int i = 0; i < fes->GetNE(); i++)
    {
       const FiniteElement *fe = fes->GetFE(i);
@@ -5225,21 +5221,7 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
          lim_energy += weight;
       }
 
-      // Normalization of the surface fitting term.
-      if (surf_fit_gf)
-      {
-         Array<int> dofs;
-         Vector sigma_e;
-         surf_fit_gf->FESpace()->GetElementDofs(i, dofs);
-         surf_fit_gf->GetSubVector(dofs, sigma_e);
-         for (int s = 0; s < dofs.Size(); s++)
-         {
-            if ((*surf_fit_marker)[dofs[s]] == true)
-            {
-               surf_fit_gf_energy += sigma_e(s) * sigma_e(s);
-            }
-         }
-      }
+      // TODO: Normalization of the surface fitting term.
    }
 
    // Cases when integration is not over the target element, or when the

fem/tmop.hpp

@@ -2038,8 +2038,7 @@ protected:
    } PA;
 
    void ComputeNormalizationEnergies(const GridFunction &x,
-                                     real_t &metric_energy, real_t &lim_energy,
-                                     real_t &surf_fit_gf_energy);
+                                     real_t &metric_energy, real_t &lim_energy);
 
    void AssembleElementVectorExact(const FiniteElement &el,
                                    ElementTransformation &T,

fem/transfer.cpp

@@ -13,6 +13,7 @@
 #include "bilinearform.hpp"
 #include "pbilinearform.hpp"
 #include "../general/forall.hpp"
+#include "kernels.hpp"
 
 namespace mfem
 {
@@ -2322,6 +2323,74 @@ void Prolongation2D(const int NE, const int D1D, const int Q1D,
    });
 }
 
+template <int DLO, int DHI>
+static void SmemProlongation3D(const int NE,
+                               const Vector& localL, Vector& localH,
+                               const Array<real_t> &b, const Vector& mask)
+{
+   auto u_lo = Reshape(localL.Read(), DLO, DLO, DLO, NE);
+   auto u_hi = Reshape(localH.Write(), DHI, DHI, DHI, NE);
+   auto d_b = Reshape(b.Read(), DHI, DLO);
+   auto m_ = Reshape(mask.Read(), DHI, DHI, DHI, NE);
+
+   mfem::forall_2D(NE, DHI, DHI, [=] MFEM_HOST_DEVICE (int e)
+   {
+      // Load B into shared memory
+      MFEM_SHARED real_t s_B[DHI*DLO];
+      kernels::internal::LoadBt<DLO,DHI>(DLO,DHI,d_b,s_B);
+      const DeviceMatrix B(s_B, DHI, DLO);
+
+      MFEM_SHARED real_t s_u[DHI*DHI*DLO];
+      const DeviceCube u(s_u, DHI, DHI, DLO);
+      real_t v[DHI];
+
+      MFEM_FOREACH_THREAD(lx,x,DLO)
+      {
+         MFEM_FOREACH_THREAD(ly,y,DLO)
+         {
+            for (int hz = 0; hz < DHI; ++hz) { v[hz] = 0.0; }
+            for (int lz = 0; lz < DLO; ++lz)
+            {
+               const real_t XYZ = u_lo(lx,ly,lz,e);
+               for (int hz = 0; hz < DHI; ++hz) { v[hz] += XYZ * B(hz,lz); }
+            }
+            for (int hz = 0; hz < DHI; ++hz) { u(hz,ly,lx) = v[hz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(hz,y,DHI)
+      {
+         MFEM_FOREACH_THREAD(lx,x,DLO)
+         {
+            for (int hy = 0; hy < DHI; ++hy) { v[hy] = 0.0; }
+            for (int ly = 0; ly < DLO; ++ly)
+            {
+               const real_t zYX = u(hz,ly,lx);
+               for (int hy = 0; hy < DHI; ++hy) { v[hy] += zYX * B(hy,ly); }
+            }
+            for (int hy = 0; hy < DHI; ++hy) { u(hz,hy,lx) = v[hy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(hz,y,DHI)
+      {
+         MFEM_FOREACH_THREAD(hy,x,DHI)
+         {
+            for (int hx = 0; hx < DHI; ++hx) { v[hx] = 0.0; }
+            for (int lx = 0; lx < DLO; ++lx)
+            {
+               const real_t zyX = u(hz,hy,lx);
+               for (int hx = 0; hx < DHI; ++hx) { v[hx] += zyX * B(hx,lx); }
+            }
+            for (int hx = 0; hx < DHI; ++hx)
+            {
+               u_hi(hx,hy,hz,e) = m_(hx,hy,hz,e)*v[hx];
+            }
+         }
+      }
+   });
+}
+
 void Prolongation3D(const int NE, const int D1D, const int Q1D,
                     const Vector& localL, Vector& localH,
                     const Array<real_t>& B, const Vector& mask)
@@ -2403,9 +2472,9 @@ void Prolongation3D(const int NE, const int D1D, const int Q1D,
    });
 }
 
-void Restriction2D(const int NE, const int D1D, const int Q1D,
-                   const Vector& localH, Vector& localL,
-                   const Array<real_t>& Bt, const Vector& mask)
+void ProlongationTranspose2D(const int NE, const int D1D, const int Q1D,
+                             const Vector& localH, Vector& localL,
+                             const Array<real_t>& Bt, const Vector& mask)
 {
    auto x_ = Reshape(localH.Read(), Q1D, Q1D, NE);
    auto y_ = Reshape(localL.Write(), D1D, D1D, NE);
@@ -2448,9 +2517,78 @@ void Restriction2D(const int NE, const int D1D, const int Q1D,
       }
    });
 }
-void Restriction3D(const int NE, const int D1D, const int Q1D,
-                   const Vector& localH, Vector& localL,
-                   const Array<real_t>& Bt, const Vector& mask)
+
+template <int DLO, int DHI>
+static void SmemProlongationTranspose3D(
+   const int NE, const Vector& localH, Vector& localL,
+   const Array<real_t>& bt, const Vector& mask)
+{
+   auto u_h = Reshape(localH.Read(), DHI, DHI, DHI, NE);
+   auto u_l = Reshape(localL.Write(), DLO, DLO, DLO, NE);
+   auto d_bt = Reshape(bt.Read(), DLO, DHI);
+   auto m_ = Reshape(mask.Read(), DHI, DHI, DHI, NE);
+
+   mfem::forall_2D(NE, DHI, DHI, [=] MFEM_HOST_DEVICE (int e)
+   {
+      // Load Bt into shared memory
+      MFEM_SHARED real_t s_Bt[DHI*DLO];
+      kernels::internal::LoadBt<DHI,DLO>(DHI,DLO,d_bt,s_Bt);
+      const DeviceMatrix Bt(s_Bt, DLO, DHI);
+
+      MFEM_SHARED real_t s_u[DLO*DHI*DHI];
+      const DeviceCube u(s_u, DLO, DHI, DHI);
+      real_t v[DLO];
+
+      MFEM_FOREACH_THREAD(hx,x,DHI)
+      {
+         MFEM_FOREACH_THREAD(hy,y,DHI)
+         {
+            for (int lz = 0; lz < DLO; ++lz) { v[lz] = 0.0; }
+            for (int hz = 0; hz < DHI; ++hz)
+            {
+               const real_t XYZ = m_(hx,hy,hz,e)*u_h(hx,hy,hz,e);
+               for (int lz = 0; lz < DLO; ++lz) { v[lz] += XYZ * Bt(lz,hz); }
+            }
+            for (int lz = 0; lz < DLO; ++lz) { u(lz,hy,hx) = v[lz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(lz,y,DLO)
+      {
+         MFEM_FOREACH_THREAD(hx,x,DHI)
+         {
+            for (int ly = 0; ly < DLO; ++ly) { v[ly] = 0.0; }
+            for (int hy = 0; hy < DHI; ++hy)
+            {
+               const real_t zYX = u(lz,hy,hx);
+               for (int ly = 0; ly < DLO; ++ly) { v[ly] += zYX * Bt(ly,hy); }
+            }
+            for (int ly = 0; ly < DLO; ++ly) { u(lz,ly,hx) = v[ly]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(lz,y,DLO)
+      {
+         MFEM_FOREACH_THREAD(ly,x,DLO)
+         {
+            for (int lx = 0; lx < DLO; ++lx) { v[lx] = 0.0; }
+            for (int hx = 0; hx < DHI; ++hx)
+            {
+               const real_t zyX = u(lz,ly,hx);
+               for (int lx = 0; lx < DLO; ++lx) { v[lx] += zyX * Bt(lx,hx); }
+            }
+            for (int lx = 0; lx < DLO; ++lx)
+            {
+               u_l(lx,ly,lz,e) = v[lx];
+            }
+         }
+      }
+   });
+}
+
+void ProlongationTranspose3D(const int NE, const int D1D, const int Q1D,
+                             const Vector& localH, Vector& localL,
+                             const Array<real_t>& Bt, const Vector& mask)
 {
    auto x_ = Reshape(localH.Read(), Q1D, Q1D, Q1D, NE);
    auto y_ = Reshape(localL.Write(), D1D, D1D, D1D, NE);
@@ -2518,11 +2656,14 @@ void Restriction3D(const int NE, const int D1D, const int Q1D,
       }
    });
 }
+
 } // namespace TransferKernels
 
 void TensorProductPRefinementTransferOperator::Mult(const Vector& x,
                                                     Vector& y) const
 {
+   using namespace TransferKernels;
+
    if (lFESpace.GetMesh()->GetNE() == 0)
    {
       return;
@@ -2531,11 +2672,25 @@ void TensorProductPRefinementTransferOperator::Mult(const Vector& x,
    elem_restrict_lex_l->Mult(x, localL);
    if (dim == 2)
    {
-      TransferKernels::Prolongation2D(NE, D1D, Q1D, localL, localH, B, mask);
+      Prolongation2D(NE, D1D, Q1D, localL, localH, B, mask);
    }
    else if (dim == 3)
    {
-      TransferKernels::Prolongation3D(NE, D1D, Q1D, localL, localH, B, mask);
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23:
+            SmemProlongation3D<2,3>(NE, localL, localH, B, mask); break;
+         case 0x24:
+            SmemProlongation3D<2,4>(NE, localL, localH, B, mask); break;
+         case 0x35:
+            SmemProlongation3D<3,5>(NE, localL, localH, B, mask); break;
+         case 0x46:
+            SmemProlongation3D<4,6>(NE, localL, localH, B, mask); break;
+         case 0x47:
+            SmemProlongation3D<4,7>(NE, localL, localH, B, mask); break;
+         default:
+            Prolongation3D(NE, D1D, Q1D, localL, localH, B, mask); break;
+      }
    }
    else
    {
@@ -2549,6 +2704,8 @@ void TensorProductPRefinementTransferOperator::Mult(const Vector& x,
 void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
                                                              Vector& y) const
 {
+   using namespace TransferKernels;
+
    if (lFESpace.GetMesh()->GetNE() == 0)
    {
       return;
@@ -2557,11 +2714,25 @@ void TensorProductPRefinementTransferOperator::MultTranspose(const Vector& x,
    elem_restrict_lex_h->Mult(x, localH);
    if (dim == 2)
    {
-      TransferKernels::Restriction2D(NE, D1D, Q1D, localH, localL, Bt, mask);
+      ProlongationTranspose2D(NE, D1D, Q1D, localH, localL, Bt, mask);
    }
    else if (dim == 3)
    {
-      TransferKernels::Restriction3D(NE, D1D, Q1D, localH, localL, Bt, mask);
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23:
+            SmemProlongationTranspose3D<2,3>(NE, localH, localL, Bt, mask); break;
+         case 0x24:
+            SmemProlongationTranspose3D<2,4>(NE, localH, localL, Bt, mask); break;
+         case 0x35:
+            SmemProlongationTranspose3D<3,5>(NE, localH, localL, Bt, mask); break;
+         case 0x46:
+            SmemProlongationTranspose3D<4,6>(NE, localH, localL, Bt, mask); break;
+         case 0x47:
+            SmemProlongationTranspose3D<4,7>(NE, localH, localL, Bt, mask); break;
+         default:
+            ProlongationTranspose3D(NE, D1D, Q1D, localH, localL, Bt, mask); break;
+      }
    }
    else
    {

fem/transfer.hpp

@@ -621,9 +621,6 @@ public:
       const FiniteElementSpace& lFESpace_,
       const FiniteElementSpace& hFESpace_);
 
-   /// Destructor
-   virtual ~TensorProductPRefinementTransferOperator() { }
-
    /// @brief Interpolation or prolongation of a vector \p x corresponding to
    /// the coarse space to the vector \p y corresponding to the fine space.
    void Mult(const Vector& x, Vector& y) const override;

general/annotation.hpp

@@ -14,24 +14,111 @@
 
 #include "../config/config.hpp"
 
-#ifdef MFEM_USE_CALIPER
+#define MFEM_CONCAT_(X,Y)  X##Y
+#define MFEM_CONCAT(X,Y)  MFEM_CONCAT_(X,Y)
 
+#ifdef MFEM_USE_CALIPER
+#include "device.hpp"
+#include "backends.hpp"
+#ifdef MFEM_USE_MPI
+#include "communication.hpp"
+#endif
+#include <optional>
 #include <caliper/cali.h>
 #include <caliper/cali-manager.h>
-#define MFEM_PERF_FUNCTION CALI_CXX_MARK_FUNCTION
-#define MFEM_PERF_BEGIN(s) CALI_MARK_BEGIN(s)
-#define MFEM_PERF_END(s) CALI_MARK_END(s)
+#endif
+
+namespace mfem
+{
+
+namespace internal
+{
+
+extern int annotation_sync_stream; // defined in globals.cpp
+extern int annotation_sync_mpi; // defined in globals.cpp
+extern int annotation_enabled; // defined in globals.cpp
+
+#ifdef MFEM_USE_CALIPER
+
+inline void AnnotationSync()
+{
+   if (!annotation_enabled) { return; }
+   if (annotation_sync_stream && Device::Allows(Backend::DEVICE_MASK))
+   {
+      MFEM_STREAM_SYNC;
+   }
+#ifdef MFEM_USE_MPI
+   if (annotation_sync_mpi && Mpi::IsInitialized() && !Mpi::IsFinalized())
+   {
+      MPI_Barrier(GetGlobalMPI_Comm());
+   }
+#endif
+}
+
+struct FunctionAnnotation
+{
+   std::optional<cali::Function> cali_func;
+
+   FunctionAnnotation(const char *fname)
+   {
+      AnnotationSync();
+      if (annotation_enabled) { cali_func.emplace(fname); }
+   }
+
+   ~FunctionAnnotation() { AnnotationSync(); }
+};
+
+struct ScopeAnnotation
+{
+   std::optional<cali::ScopeAnnotation> cali_scope;
+
+   ScopeAnnotation(const char *name)
+   {
+      AnnotationSync();
+      if (annotation_enabled) { cali_scope.emplace(name); }
+   }
+
+   ~ScopeAnnotation() { AnnotationSync(); }
+};
+
+#endif // #ifdef MFEM_USE_CALIPER
+
+} // namespace internal
+
+} // namespace mfem
+
+
+#ifdef MFEM_USE_CALIPER
+
+#define MFEM_PERF_FUNCTION \
+   mfem::internal::FunctionAnnotation mfem_func_annotation_(_MFEM_FUNC_NAME)
+#define MFEM_PERF_BEGIN(s) \
+   (mfem::internal::AnnotationSync(), CALI_MARK_BEGIN(s))
+#define MFEM_PERF_END(s) \
+   (mfem::internal::AnnotationSync(), CALI_MARK_END(s))
 #define MFEM_PERF_SCOPE(name) \
-  cali::Annotation::Guard cali_autogenerated_guard_name(cali::Annotation("function").begin(std::string(name).c_str()))
+   mfem::internal::ScopeAnnotation \
+      MFEM_CONCAT(mfem_scope_annotation_,__LINE__)(name)
 
+#define MFEM_PERF_SYNC_STREAM(b) (mfem::internal::annotation_sync_stream = (b))
+#define MFEM_PERF_SYNC_MPI(b) (mfem::internal::annotation_sync_mpi = (b))
+#define MFEM_PERF_SYNC(b) (MFEM_PERF_SYNC_STREAM(b), MFEM_PERF_SYNC_MPI(b))
+#define MFEM_PERF_ENABLE (mfem::internal::annotation_enabled = 1)
+#define MFEM_PERF_DISABLE (mfem::internal::annotation_enabled = 0)
 
-#else
+#else // #ifdef MFEM_USE_CALIPER
 
 #define MFEM_PERF_FUNCTION
 #define MFEM_PERF_BEGIN(s)
 #define MFEM_PERF_END(s)
 #define MFEM_PERF_SCOPE(name)
 
-#endif
+#define MFEM_PERF_SYNC_STREAM(b)
+#define MFEM_PERF_SYNC_MPI(b)
+#define MFEM_PERF_SYNC(b)
+#define MFEM_PERF_ENABLE
+#define MFEM_PERF_DISABLE
 
-#endif
+#endif // #ifdef MFEM_USE_CALIPER
+
+#endif // MFEM_ANNOTATION_HPP

general/array.hpp

@@ -326,7 +326,11 @@ public:
        the Size to match this Capacity after this.*/
    template <typename U>
    inline void CopyFrom(const U *src)
-   { std::memcpy(begin(), src, MemoryUsage()); }
+   {
+      if (!begin() || size == 0) { return; }
+      MFEM_ASSERT(begin() && src, "Error in Array::CopyFrom");
+      std::memcpy(begin(), src, MemoryUsage());
+   }
 
    /// STL-like begin.  Returns pointer to the first element of the array.
    inline T* begin() { return data; }

general/backends.hpp

@@ -62,6 +62,7 @@
 #define MFEM_THREAD_ID(k) 0
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
+#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA

general/communication.hpp

@@ -23,13 +23,9 @@
 #include <mpi.h>
 #include <cstdint>
 
-// can't directly use MPI_CXX_BOOL because Microsoft's MPI implementation
-// doesn't include MPI_CXX_BOOL. Fallback to MPI_C_BOOL if unavailable.
-#ifdef MPI_CXX_BOOL
-#define MFEM_MPI_CXX_BOOL MPI_CXX_BOOL
-#else
-#define MFEM_MPI_CXX_BOOL MPI_C_BOOL
-#endif
+// Some MPI implementations do not have MPI_CXX_BOOL or do not handle it
+// correctly, so we use MPI_UNSIGNED_CHAR as the MPI type for 'bool':
+#define MFEM_MPI_CXX_BOOL MPI_UNSIGNED_CHAR
 
 namespace mfem
 {

general/cuda.hpp

@@ -47,6 +47,7 @@
 #define MFEM_THREAD_ID(k) threadIdx.k
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
+#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
 #endif
 
 namespace mfem

general/device.cpp

@@ -16,6 +16,7 @@
 #include "../fem/ceed/interface/util.hpp"
 #endif
 #ifdef MFEM_USE_MPI
+#include "communication.hpp"
 #include "../linalg/hypre.hpp"
 #endif
 
@@ -145,6 +146,27 @@ Device::Device()
       Configure(device);
       device_env = true;
    }
+
+   if (GetEnv("MFEM_GPU_AWARE_MPI"))
+   {
+      SetGPUAwareMPI(true);
+   }
+
+   if (const char *mfem_perf_sync = GetEnv("MFEM_PERF_SYNC"))
+   {
+      MFEM_PERF_SYNC(std::atoi(mfem_perf_sync));
+      MFEM_CONTRACT_VAR(mfem_perf_sync);
+   }
+   if (const char *mfem_perf_sync_stream = GetEnv("MFEM_PERF_SYNC_STREAM"))
+   {
+      MFEM_PERF_SYNC_STREAM(std::atoi(mfem_perf_sync_stream));
+      MFEM_CONTRACT_VAR(mfem_perf_sync_stream);
+   }
+   if (const char *mfem_perf_sync_mpi = GetEnv("MFEM_PERF_SYNC_MPI"))
+   {
+      MFEM_PERF_SYNC_MPI(std::atoi(mfem_perf_sync_mpi));
+      MFEM_CONTRACT_VAR(mfem_perf_sync_mpi);
+   }
 }
 
 Device::~Device()
@@ -196,6 +218,29 @@ void Device::Configure(const std::string &device, const int device_id)
    {
       bmap[internal::backend_name[i]] = internal::backend_list[i];
    }
+   // auto-detect GPU configurations
+   // assumes only one of HIP or CUDA are available
+#ifdef MFEM_USE_HIP
+   bmap["gpu"] = Backend::HIP;
+#ifdef MFEM_USE_RAJA
+   bmap["raja-gpu"] = Backend::RAJA_HIP;
+#endif
+#ifdef MFEM_USE_CEED
+   bmap["ceed-gpu"] = Backend::CEED_HIP;
+#endif
+   // no OCCA+HIP?
+#elif defined(MFEM_USE_CUDA)
+   bmap["gpu"] = Backend::CUDA;
+#ifdef MFEM_USE_RAJA
+   bmap["raja-gpu"] = Backend::RAJA_CUDA;
+#endif
+#ifdef MFEM_USE_CEED
+   bmap["ceed-gpu"] = Backend::CEED_CUDA;
+#endif
+#ifdef MFEM_USE_OCCA
+   bmap["occa-gpu"] = Backend::OCCA_CUDA;
+#endif
+#endif
    std::string device_option;
    std::string::size_type beg = 0, end;
    while (1)
@@ -313,6 +358,13 @@ void Device::Print(std::ostream &os)
    {
       os << ',' << MemoryTypeName[static_cast<int>(device_mem_type)];
    }
+#ifdef MFEM_USE_MPI
+   if (Allows(Backend::DEVICE_MASK) &&
+       Mpi::IsInitialized() && !Mpi::IsFinalized())
+   {
+      os << "\nUse GPU-aware MPI:    " << (GetGPUAwareMPI() ? "yes" : "no");
+   }
+#endif
    os << std::endl;
 }
 

general/device.hpp

@@ -198,6 +198,10 @@ public:
          'ceed-hip', 'hip', 'debug',
          'occa-omp', 'raja-omp', 'omp',
          'ceed-cpu', 'occa-cpu', 'raja-cpu', 'cpu'.
+       - The following backend aliases are also available: 'ceed-gpu',
+         'occa-gpu', 'raja-gpu', and 'gpu' where they alias their respective
+         '*-cuda' or '*-hip' backends depending on the MFEM build-time
+         configuration.
        - Multiple backends can be configured at the same time.
        - Only one 'occa-*' backend can be configured at a time.
        - The backend 'occa-cuda' enables the 'cuda' backend unless 'raja-cuda'

general/error.cpp

@@ -193,4 +193,4 @@ void mfem_warning(const char *msg)
    }
 }
 
-}
+} // namespace mfem

general/error.hpp

@@ -208,4 +208,4 @@ __device__ void abort_msg(T & msg)
 #define MFEM_ASSERT_KERNEL(x,...)
 #endif
 
-#endif
+#endif // MFEM_ERROR_HPP

general/globals.cpp

@@ -31,6 +31,10 @@ namespace internal
 {
 bool mfem_out_initialized = false;
 bool mfem_err_initialized = false;
+
+int annotation_sync_stream = 0; // declared in annotation.hpp
+int annotation_sync_mpi = 0; // declared in annotation.hpp
+int annotation_enabled = 1; // declared in annotation.hpp
 }
 
 void OutStream::Init()

general/hip.hpp

@@ -47,7 +47,9 @@
 #define MFEM_THREAD_ID(k) hipThreadIdx_ ##k
 #define MFEM_THREAD_SIZE(k) hipBlockDim_ ##k
 #define MFEM_FOREACH_THREAD(i,k,N) \
-    for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
+   for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
+#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
+   if(const int i=hipThreadIdx_ ##k; i<N)
 #endif
 
 namespace mfem

general/mem_manager.hpp

@@ -657,7 +657,8 @@ private: // Static methods used by the Memory<T> class
    /// Return the host pointer.
    MFEM_ENZYME_INACTIVE static void *Register_(void *ptr, void *h_ptr,
                                                size_t bytes, MemoryType mt,
-                                               bool own, bool alias, unsigned &flags);
+                                               bool own, bool alias,
+                                               unsigned &flags);
 
    /// Register a pair of external host and device pointers
    static void Register2_(void *h_ptr, void *d_ptr, size_t bytes,
@@ -741,7 +742,7 @@ private:
 
    /// Insert a host address @a h_ptr and size *a bytes in the memory map to be
    /// managed.
-   void Insert(void *h_ptr, size_t bytes, MemoryType h_mt,  MemoryType d_mt);
+   void Insert(void *h_ptr, size_t bytes, MemoryType h_mt, MemoryType d_mt);
 
    /// Insert a device and the host addresses in the memory map
    void InsertDevice(void *d_ptr, void *h_ptr, size_t bytes,
@@ -980,7 +981,7 @@ inline void Memory<T>::Wrap(T *ptr, int size, bool own)
 #ifdef MFEM_DEBUG
    if (own && MemoryManager::Exists())
    {
-      MemoryType h_ptr_mt = MemoryManager::GetHostMemoryType_(h_ptr);
+      MemoryType h_ptr_mt = MemoryManager::GetHostMemoryType_((void*)h_ptr);
       MFEM_VERIFY(h_mt == h_ptr_mt,
                   "h_mt = " << (int)h_mt << ", h_ptr_mt = " << (int)h_ptr_mt);
    }
@@ -988,7 +989,8 @@ inline void Memory<T>::Wrap(T *ptr, int size, bool own)
    if (own && h_mt != MemoryType::HOST)
    {
       const size_t bytes = size*sizeof(T);
-      MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags);
+      MemoryManager::Register_((void*)ptr, (void*)ptr, bytes, h_mt, own, false,
+                               flags);
    }
 }
 

linalg/batched/gpu_blas.cpp

@@ -20,13 +20,13 @@
 #define MFEM_CU_or_HIP(stub) HIP##stub
 #endif
 
-#define MFEM_CONCAT(x, y, z) MFEM_CONCAT_(x, y, z)
-#define MFEM_CONCAT_(x, y, z) x ## y ## z
+#define MFEM_CONCAT3(x, y, z) MFEM_CONCAT3_(x, y, z)
+#define MFEM_CONCAT3_(x, y, z) x ## y ## z
 
 #ifdef MFEM_USE_SINGLE
-#define MFEM_GPUBLAS_PREFIX(stub) MFEM_CONCAT(MFEM_cu_or_hip(blas), S, stub)
+#define MFEM_GPUBLAS_PREFIX(stub) MFEM_CONCAT3(MFEM_cu_or_hip(blas), S, stub)
 #elif defined(MFEM_USE_DOUBLE)
-#define MFEM_GPUBLAS_PREFIX(stub) MFEM_CONCAT(MFEM_cu_or_hip(blas), D, stub)
+#define MFEM_GPUBLAS_PREFIX(stub) MFEM_CONCAT3(MFEM_cu_or_hip(blas), D, stub)
 #endif
 
 #define MFEM_BLAS_SUCCESS MFEM_CU_or_HIP(BLAS_STATUS_SUCCESS)

linalg/hypre.cpp

@@ -2574,6 +2574,18 @@ void HypreParMatrix::EliminateBC(const Array<int> &ess_dofs,
 #if defined(HYPRE_USING_GPU)
       if (HypreUsingGPU())
       {
+#if defined(HYPRE_WITH_GPU_AWARE_MPI) || defined(HYPRE_USING_GPU_AWARE_MPI)
+         // hypre_GetGpuAwareMPI() was introduced in v2.31.0, however, its value
+         // is not checked in hypre_ParCSRCommHandleCreate_v2() before v2.33.0,
+         // instead only HYPRE_WITH_GPU_AWARE_MPI is checked.
+#if MFEM_HYPRE_VERSION >= 23300
+         if (hypre_GetGpuAwareMPI())
+#endif
+         {
+            // ensure int_buf_data has been computed before sending it
+            MFEM_STREAM_SYNC;
+         }
+#endif
          // Try to use device-aware MPI for the communication if available
          comm_handle = hypre_ParCSRCommHandleCreate_v2(
                           11, comm_pkg, HYPRE_MEMORY_DEVICE, int_buf_data,
@@ -4079,6 +4091,8 @@ void HypreSolver::Setup(const HypreParVector &b, HypreParVector &x) const
 {
    if (setup_called) { return; }
 
+   MFEM_PERF_FUNCTION;
+
    MFEM_VERIFY(A != NULL, "HypreParMatrix A is missing");
 
    HYPRE_Int err_flag = SetupFcn()(*this, *A, b, x);
@@ -4104,6 +4118,8 @@ void HypreSolver::Setup(const Vector &b, Vector &x) const
 
 void HypreSolver::Mult(const HypreParVector &b, HypreParVector &x) const
 {
+   MFEM_PERF_FUNCTION;
+
    HYPRE_Int err_flag;
    if (A == NULL)
    {

linalg/operator.cpp

@@ -50,17 +50,19 @@ void Operator::InitTVectors(const Operator *Po, const Operator *Ri,
 
 void Operator::AddMult(const Vector &x, Vector &y, const real_t a) const
 {
-   mfem::Vector z(y.Size());
-   Mult(x, z);
-   y.Add(a, z);
+   z_am.SetSize(y.Size());
+   z_am.UseDevice(true);
+   Mult(x, z_am);
+   y.Add(a, z_am);
 }
 
 void Operator::AddMultTranspose(const Vector &x, Vector &y,
                                 const real_t a) const
 {
-   mfem::Vector z(y.Size());
-   MultTranspose(x, z);
-   y.Add(a, z);
+   z_am.SetSize(y.Size());
+   z_am.UseDevice(true);
+   MultTranspose(x, z_am);
+   y.Add(a, z_am);
 }
 
 void Operator::ArrayMult(const Array<const Vector *> &X,

linalg/operator.hpp

@@ -23,6 +23,13 @@ class RectangularConstrainedOperator;
 /// Abstract operator
 class Operator
 {
+private:
+   /// Auxiliary Vector used by the methods AddMult() and AddMultTranspose().
+   /** @note This Vector is private to prevent derived classes from accidentaly
+       using it in their implementation of Mult() or MultTranspose() which may
+       lead to hard-to-find bugs. */
+   mutable Vector z_am;
+
 protected:
    int height; ///< Dimension of the output / number of rows in the matrix.
    int width;  ///< Dimension of the input / number of columns in the matrix.

linalg/solvers.cpp

@@ -314,25 +314,29 @@ void OperatorJacobiSmoother::Mult(const Vector &x, Vector &y) const
    MFEM_VERIFY(x.Size() == Width(), "invalid input vector");
    MFEM_VERIFY(y.Size() == Height(), "invalid output vector");
 
+   auto DI = dinv.Read();
+   auto X = x.Read();
    if (iterative_mode)
    {
       MFEM_VERIFY(oper, "iterative_mode == true requires the forward operator");
       oper->Mult(y, residual);  // r = A y
-      subtract(x, residual, residual); // r = x - A y
+      auto R = residual.Read();
+      auto Y = y.ReadWrite();
+      // y += D^{-1} (x - A y)
+      mfem::forall(height, [=] MFEM_HOST_DEVICE (int i)
+      {
+         Y[i] += DI[i] * (X[i] - R[i]);
+      });
    }
    else
    {
-      residual = x;
-      y.UseDevice(true);
-      y = 0.0;
+      auto Y = y.Write();
+      // y = D^{-1} x
+      mfem::forall(height, [=] MFEM_HOST_DEVICE (int i)
+      {
+         Y[i] = DI[i] * X[i];
+      });
    }
-   auto DI = dinv.Read();
-   auto R = residual.Read();
-   auto Y = y.ReadWrite();
-   mfem::forall(height, [=] MFEM_HOST_DEVICE (int i)
-   {
-      Y[i] += DI[i] * R[i];
-   });
 }
 
 OperatorChebyshevSmoother::OperatorChebyshevSmoother(const Operator &oper_,
@@ -348,7 +352,8 @@ OperatorChebyshevSmoother::OperatorChebyshevSmoother(const Operator &oper_,
    diag(d),
    coeffs(order),
    ess_tdof_list(ess_tdofs),
-   residual(N),
+   residual(order > 1 ? N : 0),
+   z(order > 1 ? N : 0),
    oper(&oper_) { Setup(); }
 
 #ifdef MFEM_USE_MPI
@@ -375,7 +380,8 @@ OperatorChebyshevSmoother::OperatorChebyshevSmoother(const Operator &oper_,
      diag(d),
      coeffs(order),
      ess_tdof_list(ess_tdofs),
-     residual(N),
+     residual(order > 1 ? N : 0),
+     z(order > 1 ? N : 0),
      oper(&oper_)
 {
    OperatorJacobiSmoother invDiagOperator(diag, ess_tdofs, 1.0);
@@ -422,7 +428,7 @@ void OperatorChebyshevSmoother::Setup()
 {
    // Invert diagonal
    residual.UseDevice(true);
-   helperVector.UseDevice(true);
+   z.UseDevice(true);
    auto D = diag.Read();
    auto X = dinv.Write();
    mfem::forall(N, [=] MFEM_HOST_DEVICE (int i) { X[i] = 1.0 / D[i]; });
@@ -432,6 +438,20 @@ void OperatorChebyshevSmoother::Setup()
       X[I[i]] = 1.0;
    });
 
+   const int order_save = order;
+   order = -1; // avoid early exit in SetOrder() when 'new_order' == 'order'
+   SetOrder(order_save);
+}
+
+void OperatorChebyshevSmoother::SetOrder(int new_order)
+{
+   if (new_order == order) { return; }
+
+   order = new_order;
+   coeffs.SetSize(order);
+   residual.SetSize(order > 1 ? N : 0);
+   z.SetSize(order > 1 ? N : 0);
+
    // Set up Chebyshev coefficients
    // For reference, see e.g., Parallel multigrid smoothing: polynomial versus
    // Gauss-Seidel by Adams et al.
@@ -511,32 +531,55 @@ void OperatorChebyshevSmoother::Mult(const Vector& x, Vector &y) const
       MFEM_ABORT("Chebyshev smoother requires operator");
    }
 
-   residual = x;
-   helperVector.SetSize(x.Size());
-   helperVector.UseDevice(true);
-
-   y.UseDevice(true);
-   y = 0.0;
-
-   for (int k = 0; k < order; ++k)
+   // for k = 0, perform:
+   //    r = D^{-1} x
+   //    y = C_0 r
+   const real_t C_0 = coeffs[0];
+   auto Dinv = dinv.Read();
+   auto X = x.Read();
+   auto Y0 = y.Write();
+   if (order == 1)
    {
-      // Apply
-      if (k > 0)
+      mfem::forall(N, [=] MFEM_HOST_DEVICE (int i)
       {
-         oper->Mult(residual, helperVector);
-         residual = helperVector;
-      }
+         Y0[i] = C_0 * Dinv[i] * X[i];
+      });
+   }
+   else
+   {
+      auto R0 = residual.Write();
+      mfem::forall(N, [=] MFEM_HOST_DEVICE (int i)
+      {
+         Y0[i] = C_0 * (R0[i] = Dinv[i] * X[i]);
+      });
+   }
 
-      // Scale residual by inverse diagonal
-      const int n = N;
-      auto Dinv = dinv.Read();
-      auto R = residual.ReadWrite();
-      mfem::forall(n, [=] MFEM_HOST_DEVICE (int i) { R[i] *= Dinv[i]; });
+   for (int k = 1; k < order; ++k)
+   {
+      // Apply: z = A r
+      oper->Mult(residual, z);
 
-      // Add weighted contribution to y
+      // Scale residual by inverse diagonal and add weighted contribution to y:
+      //   r = D^{-1} z
+      //   y += C_k r
+      const real_t C_k = coeffs[k];
+      auto Z = z.Read();
       auto Y = y.ReadWrite();
-      auto C = coeffs.Read();
-      mfem::forall(n, [=] MFEM_HOST_DEVICE (int i) { Y[i] += C[k] * R[i]; });
+      if (k < order-1)
+      {
+         auto R = residual.Write();
+         mfem::forall(N, [=] MFEM_HOST_DEVICE (int i)
+         {
+            Y[i] += C_k * (R[i] = Dinv[i] * Z[i]);
+         });
+      }
+      else
+      {
+         mfem::forall(N, [=] MFEM_HOST_DEVICE (int i)
+         {
+            Y[i] += C_k * Dinv[i] * Z[i];
+         });
+      }
    }
 }
 
@@ -3213,7 +3256,7 @@ void ResidualBCMonitor::MonitorResidual(
    MPI_Comm comm = iter_solver->GetComm();
    if (comm != MPI_COMM_NULL)
    {
-      double glob_bc_norm_squared = 0.0;
+      real_t glob_bc_norm_squared = 0.0;
       MPI_Reduce(&bc_norm_squared, &glob_bc_norm_squared, 1,
                  MPITypeMap<real_t>::mpi_type,
                  MPI_SUM, 0, comm);

linalg/solvers.hpp

@@ -380,11 +380,11 @@ public:
    void SetPositiveDiagonal(bool pos_diag = true) { use_abs_diag = pos_diag; }
 
    /// Approach the solution of the linear system by applying Jacobi smoothing.
-   void Mult(const Vector &x, Vector &y) const;
+   void Mult(const Vector &x, Vector &y) const override;
 
    /** @brief Approach the solution of the transposed linear system by applying
        Jacobi smoothing. */
-   void MultTranspose(const Vector &x, Vector &y) const { Mult(x, y); }
+   void MultTranspose(const Vector &x, Vector &y) const override { Mult(x, y); }
 
    /** @brief Recompute the diagonal using the method AssembleDiagonal of the
        given new Operator, @a op. */
@@ -397,7 +397,7 @@ public:
        When the new Operator, @a op, is not a (Par)BilinearForm, any previously
        set array of essential true-dofs will be thrown away because in this case
        any essential b.c. will be handled by the AssembleDiagonal method. */
-   void SetOperator(const Operator &op);
+   void SetOperator(const Operator &op) override;
 
 private:
    Vector dinv;
@@ -481,21 +481,22 @@ public:
 
    /** @brief Approach the solution of the linear system by applying Chebyshev
        smoothing. */
-   void Mult(const Vector &x, Vector &y) const;
+   void Mult(const Vector &x, Vector &y) const override;
 
    /** @brief Approach the solution of the transposed linear system by applying
        Chebyshev smoothing. */
-   void MultTranspose(const Vector &x, Vector &y) const { Mult(x, y); }
+   void MultTranspose(const Vector &x, Vector &y) const override { Mult(x, y); }
 
-   void SetOperator(const Operator &op_)
+   void SetOperator(const Operator &op_) override
    {
       oper = &op_;
    }
 
    void Setup();
+   void SetOrder(int new_order);
 
 private:
-   const int order;
+   int order;
    real_t max_eig_estimate;
    const int N;
    Vector dinv;
@@ -503,7 +504,7 @@ private:
    Array<real_t> coeffs;
    const Array<int>& ess_tdof_list;
    mutable Vector residual;
-   mutable Vector helperVector;
+   mutable Vector z;
    const Operator* oper;
 };
 

makefile

@@ -125,7 +125,8 @@ EXAMPLE_TEST_DIRS := examples
 
 MINIAPP_SUBDIRS = common electromagnetics meshing navier performance tools \
  toys nurbs gslib adjoint solvers shifted mtop parelag tribol autodiff dfem \
- hooke multidomain dpg hdiv-linear-solver spde diag-smoothers
+ hooke multidomain dpg hdiv-linear-solver spde diag-smoothers \
+ benchmarks/ceed-solver-bps
 MINIAPP_DIRS := $(addprefix miniapps/,$(MINIAPP_SUBDIRS))
 MINIAPP_TEST_DIRS := $(filter-out %/common,$(MINIAPP_DIRS))
 MINIAPP_USE_COMMON := $(addprefix miniapps/,electromagnetics meshing tools \
@@ -145,7 +146,7 @@ MFEM_BUILD_DIR ?= .
 BUILD_DIR := $(MFEM_BUILD_DIR)
 BUILD_REAL_DIR := $(abspath $(BUILD_DIR))
 ifneq ($(BUILD_REAL_DIR),$(MFEM_REAL_DIR))
-   BUILD_SUBDIRS = $(DIRS) config $(EM_DIRS) doc $(TEST_DIRS)
+   BUILD_SUBDIRS = $(DIRS) config $(EM_DIRS) doc $(TEST_DIRS) tests/benchmarks
    CONFIG_FILE_DEF = -DMFEM_CONFIG_FILE='"$(BUILD_REAL_DIR)/config/_config.hpp"'
    BLD := $(if $(BUILD_REAL_DIR:$(CURDIR)=),$(BUILD_DIR)/,)
    $(if $(word 2,$(BLD)),$(error Spaces in BLD = "$(BLD)" are not supported))
@@ -582,13 +583,14 @@ test-print:
 	@for dir in $(ALL_TEST_DIRS); do \
 	   $(MAKE) -j1 -C $(BLD)$${dir} test-print; done
 
-ALL_CLEAN_SUBDIRS = $(addsuffix /clean,config $(EM_DIRS) doc $(TEST_DIRS))
+ALL_CLEAN_SUBDIRS = $(addsuffix /clean,config $(EM_DIRS) doc $(TEST_DIRS) \
+ tests/benchmarks)
 .PHONY: $(ALL_CLEAN_SUBDIRS) miniapps/clean
 miniapps/clean: $(addsuffix /clean,$(MINIAPP_DIRS))
 $(ALL_CLEAN_SUBDIRS):
 	$(MAKE) -C $(BLD)$(@D) $(@F)
 
-clean: $(addsuffix /clean,$(EM_DIRS) $(TEST_DIRS))
+clean: $(addsuffix /clean,$(EM_DIRS) $(TEST_DIRS) tests/benchmarks)
 	rm -f $(addprefix $(BLD),$(foreach d,$(DIRS),$(d)/*.o))
 	rm -f $(addprefix $(BLD),$(foreach d,$(DIRS),$(d)/*~))
 	rm -rf $(addprefix $(BLD),*~ libmfem.* deps.mk)
@@ -691,7 +693,8 @@ local-config:
 .PHONY: build-config
 build-config:
 	for d in $(BUILD_SUBDIRS); do mkdir -p $(BLD)$${d}; done
-	for dir in "" $(addsuffix /,config $(EM_DIRS) doc $(TEST_DIRS)); do \
+	for dir in "" $(addsuffix /,config $(EM_DIRS) doc $(TEST_DIRS) \
+                        tests/benchmarks); do \
 	   printf "# Auto-generated file.\n%s\n%s\n" \
 	      "MFEM_DIR = $(MFEM_REAL_DIR)" \
 	      "include \$$(MFEM_DIR)/$${dir}makefile" \

mesh/exodus_writer.cpp

@@ -26,6 +26,12 @@
    }\
 }
 
+#if defined(MFEM_USE_DOUBLE)
+#define MFEM_NETCDF_REAL_T NC_DOUBLE
+#elif defined(MFEM_USE_SINGLE)
+#define MFEM_NETCDF_REAL_T NC_FLOAT
+#endif
+
 namespace mfem
 {
 
@@ -135,18 +141,18 @@ public:
    /// @brief Writes the mesh to an ExodusII file.
    /// @param fpath The path to the file.
    /// @param flags NC_CLOBBER will overwrite existing file.
-   void PrintExodusII(std::string fpath, int flags = NC_CLOBBER);
+   void PrintExodusII(const std::string &fpath, int flags = NC_CLOBBER);
 
    /// @brief Static method for writing a mesh to an ExodusII file.
    /// @param mesh The mesh to write to the file.
    /// @param fpath The path to the file.
    /// @param flags NetCDF file flags.
-   static void PrintExodusII(Mesh & mesh, std::string fpath,
+   static void PrintExodusII(Mesh & mesh, const std::string &fpath,
                              int flags = NC_CLOBBER);
 
 protected:
    /// @brief Closes any open file and creates a NetCDF file using selected flags.
-   void OpenExodusII(std::string fpath, int flags);
+   void OpenExodusII(const std::string &fpath, int flags);
 
    /// @brief Closes any open file.
    void CloseExodusII();
@@ -167,9 +173,9 @@ protected:
    std::unordered_set<int> GenerateUniqueNodeIDs();
 
    /// @brief Populates vectors with x, y, z coordinates from mesh.
-   void ExtractVertexCoordinates(std::vector<double> & coordx,
-                                 std::vector<double> & coordy,
-                                 std::vector<double> & coordz);
+   void ExtractVertexCoordinates(std::vector<real_t> &coordx,
+                                 std::vector<real_t> &coordy,
+                                 std::vector<real_t> &coordz);
 
    /// @brief Writes node connectivity for a particular block.
    /// @param block_id The block to write to the file.
@@ -187,7 +193,7 @@ protected:
    /// @brief Writes the number of elements in the mesh.
    void WriteNumOfElements();
 
-   /// @brief Writes the floating-point word size (4 == float; 8 == double).
+   /// @brief Writes the floating-point word size (sizeof(real_t)).
    void WriteFloatingPointWordSize();
 
    /// @brief Writes the API version.
@@ -291,7 +297,7 @@ private:
    std::map<int, std::vector<int>> exodusII_side_ids_for_boundary_id;
 };
 
-void Mesh::PrintExodusII(const std::string fpath)
+void Mesh::PrintExodusII(const std::string &fpath)
 {
    ExodusIIWriter::PrintExodusII(*this, fpath);
 }
@@ -362,7 +368,7 @@ void ExodusIIWriter::WriteExodusIIMeshInformation()
    WriteNodeSets();
 }
 
-void ExodusIIWriter::PrintExodusII(std::string fpath, int flags)
+void ExodusIIWriter::PrintExodusII(const std::string &fpath, int flags)
 {
    OpenExodusII(fpath, flags);
 
@@ -374,7 +380,7 @@ void ExodusIIWriter::PrintExodusII(std::string fpath, int flags)
    mfem::out << "Mesh successfully written to Exodus II file" << std::endl;
 }
 
-void ExodusIIWriter::PrintExodusII(Mesh & mesh, std::string fpath,
+void ExodusIIWriter::PrintExodusII(Mesh &mesh, const std::string &fpath,
                                    int flags)
 {
    ExodusIIWriter writer(mesh);
@@ -382,7 +388,7 @@ void ExodusIIWriter::PrintExodusII(Mesh & mesh, std::string fpath,
    writer.PrintExodusII(fpath, flags);
 }
 
-void ExodusIIWriter::OpenExodusII(std::string fpath, int flags)
+void ExodusIIWriter::OpenExodusII(const std::string &fpath, int flags)
 {
    CloseExodusII();  // Close any open files.
 
@@ -422,7 +428,7 @@ void ExodusIIWriter::WriteNumOfElements()
 
 void ExodusIIWriter::WriteFloatingPointWordSize()
 {
-   const int word_size = 8;
+   const int word_size = sizeof(real_t);
    PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_FLOATING_POINT_WORD_SIZE_LABEL,
           NC_INT, 1,
           &word_size);
@@ -430,13 +436,15 @@ void ExodusIIWriter::WriteFloatingPointWordSize()
 
 void ExodusIIWriter::WriteAPIVersion()
 {
-   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_API_VERSION_LABEL, NC_FLOAT, 1,
+   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_API_VERSION_LABEL, MFEM_NETCDF_REAL_T,
+          1,
           &ExodusIILabels::EXODUS_API_VERSION);
 }
 
 void ExodusIIWriter::WriteDatabaseVersion()
 {
-   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_DATABASE_VERSION_LABEL, NC_FLOAT, 1,
+   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_DATABASE_VERSION_LABEL,
+          MFEM_NETCDF_REAL_T, 1,
           &ExodusIILabels::EXODUS_DATABASE_VERSION);
 }
 
@@ -607,25 +615,25 @@ void ExodusIIWriter::WriteNodalCoordinates()
    DefineDimension("num_nodes", num_nodes, &num_nodes_id);
 
    // 3. Extract the nodal coordinates.
-   // NB: assume doubles (could be floats!); ndims = 1 (vector).
+   // NB: writes in format real_t (double or float); ndims = 1 (vector).
    // https://docs.unidata.ucar.edu/netcdf-c/current/group__variables.html#gac7e8662c51f3bb07d1fc6d6c6d9052c8
-   std::vector<double> coordx(num_nodes);
-   std::vector<double> coordy(num_nodes);
-   std::vector<double> coordz(mesh.Dimension() == 3 ? num_nodes : 0);
+   std::vector<real_t> coordx(num_nodes);
+   std::vector<real_t> coordy(num_nodes);
+   std::vector<real_t> coordz(mesh.Dimension() == 3 ? num_nodes : 0);
 
    ExtractVertexCoordinates(coordx, coordy, coordz);
 
    // 4. Define and put the nodal coordinates.
-   DefineAndPutVar(ExodusIILabels::EXODUS_COORDX_LABEL, NC_DOUBLE, 1,
+   DefineAndPutVar(ExodusIILabels::EXODUS_COORDX_LABEL, MFEM_NETCDF_REAL_T, 1,
                    &num_nodes_id,
                    coordx.data());
-   DefineAndPutVar(ExodusIILabels::EXODUS_COORDY_LABEL, NC_DOUBLE, 1,
+   DefineAndPutVar(ExodusIILabels::EXODUS_COORDY_LABEL, MFEM_NETCDF_REAL_T, 1,
                    &num_nodes_id,
                    coordy.data());
 
    if (mesh.Dimension() == 3)
    {
-      DefineAndPutVar(ExodusIILabels::EXODUS_COORDZ_LABEL, NC_DOUBLE, 1,
+      DefineAndPutVar(ExodusIILabels::EXODUS_COORDZ_LABEL, MFEM_NETCDF_REAL_T, 1,
                       &num_nodes_id,
                       coordz.data());
    }
@@ -770,9 +778,9 @@ void ExodusIIWriter::WriteNodeConnectivityForBlock(const int block_id)
 }
 
 
-void ExodusIIWriter::ExtractVertexCoordinates(std::vector<double> & coordx,
-                                              std::vector<double> & coordy,
-                                              std::vector<double> & coordz)
+void ExodusIIWriter::ExtractVertexCoordinates(std::vector<real_t> & coordx,
+                                              std::vector<real_t> & coordy,
+                                              std::vector<real_t> & coordz)
 {
    if (mesh.GetNodes()) // Higher-order.
    {
@@ -782,7 +790,7 @@ void ExodusIIWriter::ExtractVertexCoordinates(std::vector<double> & coordx,
       sorted_node_ids.assign(unordered_node_ids.begin(), unordered_node_ids.end());
       std::sort(sorted_node_ids.begin(), sorted_node_ids.end());
 
-      double coordinates[3];
+      real_t coordinates[3];
       for (size_t i = 0; i < sorted_node_ids.size(); i++)
       {
          int node_id = sorted_node_ids[i];
@@ -802,7 +810,7 @@ void ExodusIIWriter::ExtractVertexCoordinates(std::vector<double> & coordx,
    {
       for (int ivertex = 0; ivertex < mesh.GetNV(); ivertex++)
       {
-         double * coordinates = mesh.GetVertex(ivertex);
+         real_t *coordinates = mesh.GetVertex(ivertex);
 
          coordx[ivertex] = coordinates[0];
          coordy[ivertex] = coordinates[1];
@@ -1080,4 +1088,4 @@ void ExodusIIWriter::CheckNodalFESpaceIsSecondOrderH1() const
 
 #endif
 
-}
+}

mesh/mesh.cpp

@@ -883,6 +883,8 @@ const GeometricFactors* Mesh::GetGeometricFactors(const IntegrationRule& ir,
                                                   const int flags,
                                                   MemoryType d_mt)
 {
+   MFEM_PERF_FUNCTION;
+
    for (int i = 0; i < geom_factors.Size(); i++)
    {
       GeometricFactors *gf = geom_factors[i];
@@ -2994,6 +2996,7 @@ void Mesh::DoNodeReorder(DSTable *old_v_to_v, Table *old_elem_vert)
    const int num_edge_dofs = old_dofs.Size();
 
    // Save the original nodes
+   Nodes->HostReadWrite(); // for "(*Nodes)() = "
    const Vector onodes = *Nodes;
 
    // vertex dofs do not need to be moved
@@ -14648,6 +14651,7 @@ GeometricFactors::GeometricFactors(const GridFunction &nodes,
 void GeometricFactors::Compute(const GridFunction &nodes,
                                MemoryType d_mt)
 {
+   MFEM_PERF_FUNCTION;
 
    const FiniteElementSpace *fespace = nodes.FESpace();
    const FiniteElement *fe = fespace->GetTypicalFE();

mesh/mesh.hpp

@@ -2537,7 +2537,7 @@ public:
 
 #ifdef MFEM_USE_NETCDF
    /// @brief Export a mesh to an Exodus II file.
-   void PrintExodusII(const std::string fpath);
+   void PrintExodusII(const std::string &fpath);
 #endif
 
    /** @brief Prints the mesh with boundary elements given by the boundary of

mesh/mesh_readers.cpp

@@ -802,21 +802,11 @@ struct BufferReader : BufferReaderBase
       {
          // Each "data block" is preceded by a header that is either UInt32 or
          // UInt64. The rest of the data follows.
-         uint64_t data_size;
-         if (header_type == UINT32_HEADER)
-         {
-            uint32_t *data_size_32 = (uint32_t *)header_buf;
-            data_size = *data_size_32;
-         }
-         else
-         {
-            uint64_t *data_size_64 = (uint64_t *)header_buf;
-            data_size = *data_size_64;
-         }
-         MFEM_VERIFY(sizeof(F)*n == data_size, "AppendedData: wrong data size");
+         MFEM_VERIFY(sizeof(F)*n == ReadHeaderEntry(header_buf),
+                     "AppendedData: wrong data size");
       }
 
-      if (std::is_same<T, F>::value)
+      if (std::is_same_v<T, F>)
       {
          // Special case: no type conversions necessary, so can just memcpy
          memcpy(dest, buf, sizeof(T)*n);

mesh/nurbs.cpp

@@ -9,8 +9,13 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "mesh_headers.hpp"
-#include "../fem/fem.hpp"
+#include "nurbs.hpp"
+
+#include "point.hpp"
+#include "segment.hpp"
+#include "quadrilateral.hpp"
+#include "hexahedron.hpp"
+#include "../fem/gridfunc.hpp"
 #include "../general/text.hpp"
 
 #include <fstream>
@@ -33,6 +38,7 @@ KnotVector::KnotVector(istream &input)
 
    knot.Load(input, NumOfControlPoints + Order + 1);
    GetElements();
+   coarse = false;
 }
 
 KnotVector::KnotVector(int order, int NCP)
@@ -41,6 +47,7 @@ KnotVector::KnotVector(int order, int NCP)
    NumOfControlPoints = NCP;
    knot.SetSize(NumOfControlPoints + Order + 1);
    NumOfElements = 0;
+   coarse = false;
 
    knot = -1.;
 }
@@ -86,6 +93,7 @@ KnotVector::KnotVector(int order, const Vector& intervals,
          ++NumOfElements;
       }
    }
+   coarse = false;
 }
 
 KnotVector &KnotVector::operator=(const KnotVector &kv)
@@ -1431,8 +1439,8 @@ void NURBSPatch::DegreeElevate(int dir, int t)
    KnotVector &oldkv = *kv[dir];
    oldkv.GetElements();
 
-   NURBSPatch *newpatch = new NURBSPatch(this, dir, oldkv.GetOrder() + t,
-                                         oldkv.GetNCP() + oldkv.GetNE()*t);
+   auto *newpatch = new NURBSPatch(this, dir, oldkv.GetOrder() + t,
+                                   oldkv.GetNCP() + oldkv.GetNE()*t);
    NURBSPatch &newp  = *newpatch;
    KnotVector &newkv = *newp.GetKV(dir);
 
@@ -4587,7 +4595,7 @@ void NURBSExtension::KnotInsert(Array<Vector *> &kv)
 
             // Flip vector
             int size = pkvc[d]->Size();
-            int ns = ceil(size/2.0);
+            int ns = static_cast<int>(ceil(size/2.0));
             for (int j = 0; j < ns; j++)
             {
                real_t tmp = apb - pkvc[d]->Elem(j);
@@ -4647,7 +4655,7 @@ void NURBSExtension::KnotRemove(Array<Vector *> &kv, real_t tol)
 
             // Flip vector
             int size = pkvc[d]->Size();
-            int ns = ceil(size/2.0);
+            int ns = static_cast<int>(ceil(size/2.0));
             for (int j = 0; j < ns; j++)
             {
                real_t tmp = apb - pkvc[d]->Elem(j);

mesh/nurbs.hpp

@@ -22,7 +22,6 @@
 #include "../general/communication.hpp"
 #endif
 #include <iostream>
-#include <set>
 
 namespace mfem
 {
@@ -55,7 +54,7 @@ protected:
 
 public:
    /// Create an empty KnotVector.
-   KnotVector() { }
+   KnotVector() = default;
 
    /** @brief Create a KnotVector by reading data from stream @a input. Two
        integers are read, for order and number of control points. */
@@ -74,7 +73,7 @@ public:
        polynomial degree). Periodicity is not supported.
    */
    KnotVector(int order, const Vector& intervals,
-              const Array<int>& continuity );
+              const Array<int>& continuity);
 
    /// Copy constructor.
    KnotVector(const KnotVector &kv) { (*this) = kv; }

mesh/pmesh.cpp

@@ -3132,11 +3132,12 @@ void ParMesh::GetFaceNbrElementTransformation(
          pNodes->ParFESpace()->GetFaceNbrElementVDofs(FaceNo, vdofs);
          int n = vdofs.Size()/spaceDim;
          pointmat.SetSize(spaceDim, n);
+         pNodes->FaceNbrData().HostRead();
          for (int k = 0; k < spaceDim; k++)
          {
             for (int j = 0; j < n; j++)
             {
-               pointmat(k,j) = (pNodes->FaceNbrData())(vdofs[n*k+j]);
+               pointmat(k,j) = AsConst(pNodes->FaceNbrData())(vdofs[n*k+j]);
             }
          }
 

mesh/vtkhdf.cpp

@@ -84,7 +84,7 @@ void VTKHDF::EnsureSteps()
 }
 
 hid_t VTKHDF::EnsureDataset(hid_t f, const std::string &name, hid_t type,
-                            int ndims)
+                            Dims &dims)
 {
    const char *name_c = name.c_str();
 
@@ -94,20 +94,23 @@ hid_t VTKHDF::EnsureDataset(hid_t f, const std::string &name, hid_t type,
    if (status == 0)
    {
       // Dataset does not exist, create it.
-      Dims dims(ndims);
-      Dims maxdims(ndims, H5S_UNLIMITED);
-      const hid_t fspace = H5Screate_simple(ndims, dims, maxdims);
+      const int ndims = dims.ndims;
+      // The dataset is allowed to grow in the first dimension, but is fixed
+      // in size in all other dimesions; the maximum dataset size is same as
+      // dims, but unlimited in first dimension.
+      Dims max_dims = dims;
+      max_dims[0] = H5S_UNLIMITED;
+      const hid_t fspace = H5Screate_simple(ndims, dims, max_dims);
 
       Dims chunk(ndims);
       size_t chunk_size_bytes = 1024 * 1024 / 2; // 0.5 MB
       const size_t t_bytes = H5Tget_size(type);
       for (int i = 1; i < ndims; ++i)
       {
-         chunk[i] = 16;
-         chunk_size_bytes /= 16;
+         chunk[i] = dims[i];
+         chunk_size_bytes /= dims[i];
       }
       chunk[0] = chunk_size_bytes / t_bytes;
-      for (int i = 1; i < ndims; ++i) { chunk[i] = 16; }
       const hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
       H5Pset_chunk(dcpl, ndims, chunk);
       if (compression_level >= 0)
@@ -124,7 +127,19 @@ hid_t VTKHDF::EnsureDataset(hid_t f, const std::string &name, hid_t type,
    else if (status > 0)
    {
       // Dataset exists, open it.
-      return H5Dopen2(f, name_c, H5P_DEFAULT);
+      const hid_t d = H5Dopen2(f, name_c, H5P_DEFAULT);
+
+      // Resize the dataset, set dims to its new size.
+      Dims old_dims(dims.ndims);
+      const hid_t dspace = H5Dget_space(d);
+      const int ndims_dset = H5Sget_simple_extent_ndims(dspace);
+      MFEM_VERIFY(ndims_dset == dims.ndims, "");
+      H5Sget_simple_extent_dims(dspace, old_dims, NULL);
+      H5Sclose(dspace);
+      dims[0] += old_dims[0];
+      H5Dset_extent(d, dims);
+
+      return d;
    }
    else
    {
@@ -160,27 +175,13 @@ void VTKHDF::AppendParData(hid_t f, const std::string &name, hsize_t locsize,
                            hsize_t offset, Dims globsize, T *data)
 {
    const int ndims = globsize.ndims;
-   const hid_t d = EnsureDataset(f, name, GetTypeID<T>(), ndims);
-
-   // Resize the dataset, set dims to its new size.
-   hsize_t old_size;
-   Dims dims(ndims);
-   {
-      const hid_t dspace = H5Dget_space(d);
-      const int ndims_dset = H5Sget_simple_extent_ndims(dspace);
-      MFEM_VERIFY(ndims_dset == ndims, "");
-      H5Sget_simple_extent_dims(dspace, dims, NULL);
-      H5Sclose(dspace);
-      old_size = dims[0];
-      dims[0] += globsize[0];
-      for (int i = 1; i < ndims; ++i) { dims[i] = globsize[i]; }
-      H5Dset_extent(d, dims);
-   }
+   Dims dims = globsize;
+   const hid_t d = EnsureDataset(f, name, GetTypeID<T>(), dims);
 
    // Write the new entry.
    const hid_t dspace = H5Dget_space(d);
    Dims start(ndims);
-   start[0] = old_size + offset;
+   start[0] = dims[0] - globsize[0] + offset;
    Dims count(ndims);
    count[0] = locsize;
    for (int i = 1; i < ndims; ++i) { count[i] = globsize[i]; }
@@ -334,14 +335,14 @@ void VTKHDF::Truncate(const real_t t)
    }
 
    // Index of found time index (may be 'one-past-the-end' if not found)
-   const int i = std::distance(tvals.begin(), it);
+   const ptrdiff_t i = std::distance(tvals.begin(), it);
 
    // Only truncate if needed
    const bool truncate = it != tvals.end();
 
    // Number of steps we are keeping
    nsteps = i;
-   H5LTset_attribute_int(vtk, "Steps", "NSteps", &nsteps, 1);
+   H5LTset_attribute_ulong(vtk, "Steps", "NSteps", &nsteps, 1);
 
    // We want to continue writing immediately after step 'i - 1'. If i = 0,
    // then this is at the beginning of the file, and the offsets do not need
@@ -509,7 +510,7 @@ void VTKHDF::UpdateSteps(real_t t)
 
    // Set the NSteps attribute
    ++nsteps;
-   H5LTset_attribute_int(steps, ".", "NSteps", &nsteps, 1);
+   H5LTset_attribute_ulong(steps, ".", "NSteps", &nsteps, 1);
 
    AppendValue(steps, "Values", t);
    AppendValue(steps, "PartOffsets", part_offset);
@@ -618,16 +619,16 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
 
          for (int i = 0; i < pmat.Width(); i++)
          {
-            points.push_back(pmat(0,i));
-            if (pmat.Height() > 1) { points.push_back(pmat(1,i)); }
+            points.push_back(FP_T(pmat(0,i)));
+            if (pmat.Height() > 1) { points.push_back(FP_T(pmat(1,i))); }
             else { points.push_back(0.0); }
-            if (pmat.Height() > 2) { points.push_back(pmat(2,i)); }
+            if (pmat.Height() > 2) { points.push_back(FP_T(pmat(2,i))); }
             else { points.push_back(0.0); }
          }
       }
    }
 
-   const hsize_t ne_0 = mesh.GetNE();
+   const int ne_0 = mesh.GetNE();
    const hsize_t ne = high_order ? ne_0 : ne_ref;
 
    AppendParData(vtk, "NumberOfPoints", 1, mpi_rank, mpi_dims, &np);
@@ -657,7 +658,7 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          if (high_order)
          {
             Array<int> local_connectivity;
-            for (size_t e = 0; e < ne; ++e)
+            for (int e = 0; e < int(ne); ++e)
             {
                offsets[e] = off;
                const Geometry::Type geom = mesh.GetElementGeometry(e);
@@ -675,7 +676,7 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          {
             int off_0 = 0;
             int e_ref = 0;
-            for (hsize_t e = 0; e < ne_0; ++e)
+            for (int e = 0; e < ne_0; ++e)
             {
                const Geometry::Type geom = mesh.GetElementGeometry(e);
                const int nv = get_nv(e);
@@ -714,12 +715,13 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          const int *vtk_geom_map =
             high_order ? VTKGeometry::HighOrderMap : VTKGeometry::Map;
          int e_ref = 0;
-         for (hsize_t e = 0; e < ne_0; ++e)
+         for (int e = 0; e < ne_0; ++e)
          {
-            const int ne_ref = get_ne_ref(e, ref_0);
-            for (int i = 0; i < ne_ref; ++i, ++e_ref)
+            const int ne_ref_e = get_ne_ref(e, ref_0);
+            for (int i = 0; i < ne_ref_e; ++i, ++e_ref)
             {
-               cell_types[e_ref] = vtk_geom_map[mesh.GetElementGeometry(e)];
+               cell_types[e_ref] = static_cast<unsigned char>(
+                                      vtk_geom_map[mesh.GetElementGeometry(e)]);
             }
          }
          AppendParData(vtk, "Types", ne, e_offset, Dims({ne_total}),
@@ -732,11 +734,11 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          EnsureGroup("CellData", cell_data);
          std::vector<int> attributes(ne);
          hsize_t e_ref = 0;
-         for (hsize_t e = 0; e < ne_0; ++e)
+         for (int e = 0; e < ne_0; ++e)
          {
             const int attr = mesh.GetAttribute(e);
-            const int ne_ref = get_ne_ref(e, ref_0);
-            for (int i = 0; i < ne_ref; ++i, ++e_ref)
+            const int ne_ref_e = get_ne_ref(e, ref_0);
+            for (int i = 0; i < ne_ref_e; ++i, ++e_ref)
             {
                attributes[e_ref] = attr;
             }
@@ -772,7 +774,7 @@ void VTKHDF::SaveGridFunction(const GridFunction &gf, const std::string &name)
       {
          for (int vd = 0; vd < vdim; ++vd)
          {
-            point_values[off] = vec_val(vd, i);
+            point_values[off] = FP_T(vec_val(vd, i));
             ++off;
          }
       }

mesh/vtkhdf.hpp

@@ -76,14 +76,14 @@ private:
    /// Wrapper for storing dataset dimensions (max ndims is 2D in VTKHDF).
    struct Dims
    {
-      static constexpr int MAX_NDIMS = 2;
+      static constexpr size_t MAX_NDIMS = 2;
       std::array<hsize_t, MAX_NDIMS> data = { }; // Zero initialized
       int ndims = 0;
       Dims() = default;
       Dims(int ndims_) : ndims(ndims_) { MFEM_ASSERT(ndims <= MAX_NDIMS, ""); }
       Dims(int ndims_, hsize_t val) : Dims(ndims_) { data.fill(val); }
       template <typename T>
-      Dims(std::initializer_list<T> data_) : Dims(data_.size())
+      Dims(std::initializer_list<T> data_) : Dims(int(data_.size()))
       { std::copy(data_.begin(), data_.end(), data.begin()); }
       operator hsize_t*() { return data.data(); }
       hsize_t &operator[](int i) { return data[i]; }
@@ -97,7 +97,7 @@ private:
    hid_t steps = H5I_INVALID_HID;
 
    /// Number of time steps saved.
-   int nsteps = 0;
+   unsigned long nsteps = 0;
 
    /// Keep track of the offsets into the data arrays at each time step.
    struct Offsets
@@ -123,8 +123,8 @@ private:
    class MeshId
    {
       const Mesh *mesh_ptr = nullptr;
-      int sequence = -1;
-      int nodes_sequence = -1;
+      long sequence = -1;
+      long nodes_sequence = -1;
       bool high_order = true;
       int ref = -1;
    public:
@@ -187,8 +187,10 @@ private:
    /// The rank (number of dimensions) of the dataset is given by @a ndims and
    /// its data type is given by @a type.
    ///
-   /// The dataset will initially have zero size and unlimited maximum size.
-   hid_t EnsureDataset(hid_t f, const std::string &name, hid_t type, int ndims);
+   /// If the dataset does not exist, it will initially have size @a dims.
+   /// Otherwise, it will be resized to append data of size @a dims, and @a dims
+   /// will be set to the new total size.
+   hid_t EnsureDataset(hid_t f, const std::string &name, hid_t type, Dims &dims);
 
    /// @brief Ensure the named group is open, creating it if needed. Set @a
    /// group to the ID.

miniapps/benchmarks/ceed-solver-bps/kershaw.hpp

@@ -0,0 +1,156 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef __KERSHAW_HPP__
+#define __KERSHAW_HPP__
+
+#include "mfem.hpp"
+
+namespace mfem
+{
+
+// 1D transformation at the right boundary.
+real_t right(const real_t eps, const real_t x)
+{
+   return (x <= 0.5) ? (2-eps) * x : 1 + eps*(x-1);
+}
+
+// 1D transformation at the left boundary
+real_t left(const real_t eps, const real_t x)
+{
+   return 1-right(eps,1-x);
+}
+
+// Transition from a value of "a" for x=0, to a value of "b" for x=1. Smoothness
+// is controlled by the parameter "s", taking values 0, 1, or 2.
+real_t step(const real_t a, const real_t b, real_t x, int s)
+{
+   if (x <= 0) { return a; }
+   if (x >= 1) { return b; }
+   switch (s)
+   {
+      case 0:
+      default:
+         return a + (b-a) * (x);
+      case 1: return a + (b-a) * (x*x*(3-2*x));
+      case 2: return a + (b-a) * (x*x*x*(x*(6*x-15)+10));
+   }
+}
+
+// 3D version of a generalized Kershaw mesh transformation, see D. Kershaw,
+// "Differencing of the diffusion equation in Lagrangian hydrodynamic codes",
+// JCP, 39:375–395, 1981.
+//
+// The input mesh should be Cartesian nx x ny x nz with nx divisible by 6 and
+// ny, nz divisible by 2.
+//
+// The eps parameters are in (0, 1]. Uniform mesh is recovered for epsy=epsz=1.
+void kershaw(const real_t epsy, const real_t epsz, const int smoothness,
+             const real_t x, const real_t y, const real_t z,
+             real_t &X, real_t &Y, real_t &Z)
+{
+   X = x;
+
+   int layer = x*6.0;
+   real_t lambda = (x-layer/6.0)*6;
+
+   // The x-range is split in 6 layers going from left-to-left, left-to-right,
+   // right-to-left (2 layers), left-to-right and right-to-right yz-faces.
+   switch (layer)
+   {
+      case 0:
+         Y = left(epsy, y);
+         Z = left(epsz, z);
+         break;
+      case 1:
+      case 4:
+         Y = step(left(epsy, y), right(epsy, y), lambda, smoothness);
+         Z = step(left(epsz, z), right(epsz, z), lambda, smoothness);
+         break;
+      case 2:
+         Y = step(right(epsy, y), left(epsy, y), lambda/2, smoothness);
+         Z = step(right(epsz, z), left(epsz, z), lambda/2, smoothness);
+         break;
+      case 3:
+         Y = step(right(epsy, y), left(epsy, y), (1+lambda)/2, smoothness);
+         Z = step(right(epsz, z), left(epsz, z), (1+lambda)/2, smoothness);
+         break;
+      default:
+         Y = right(epsy, y);
+         Z = right(epsz, z);
+         break;
+   }
+}
+
+struct KershawTransformation : VectorCoefficient
+{
+   real_t epsy, epsz;
+   int dim, s;
+   KershawTransformation(int dim_, real_t epsy_, real_t epsz_, int s_=0)
+      : VectorCoefficient(dim_), epsy(epsy_), epsz(epsz_), dim(dim_), s(s_) { }
+   using VectorCoefficient::Eval;
+   void Eval(Vector &V, ElementTransformation &T,
+             const IntegrationPoint &ip) override
+   {
+      real_t xyz[3];
+      Vector transip(xyz, 3);
+      T.Transform(ip, transip);
+      if (dim == 1)
+      {
+         V[0] = xyz[0]; // no transformation in 1D
+      }
+      else if (dim == 2)
+      {
+         real_t z=0, zt;
+         kershaw(epsy, epsz, s, xyz[0], xyz[1], z, V[0], V[1], zt);
+      }
+      else // dim == 3
+      {
+         kershaw(epsy, epsz, s, xyz[0], xyz[1], xyz[2], V[0], V[1], V[2]);
+      }
+   }
+};
+
+ParMesh CreateKershawMesh(int nx, int ny, int nz, real_t epsy, real_t epsz)
+{
+   const bool sfc_order = true;
+   Mesh serial_mesh;
+   if (nx > 0 && ny == 0 && nz == 0)
+   {
+      serial_mesh = Mesh::MakeCartesian1D(nx, 1.0);
+   }
+   else if (nx > 0 && ny > 0 && nz == 0)
+   {
+      serial_mesh = Mesh::MakeCartesian2D(nx, ny, Element::QUADRILATERAL,
+                                          false, 1, 1, sfc_order);
+   }
+   else if (nx > 0 && ny > 0 && nz > 0)
+   {
+      serial_mesh = Mesh::MakeCartesian3D(nx, ny, nz, Element::HEXAHEDRON,
+                                          1, 1, 1, sfc_order);
+   }
+   else
+   {
+      MFEM_ABORT("Bad grid size");
+   }
+   KershawTransformation kt(serial_mesh.Dimension(), epsy, epsz);
+   serial_mesh.Transform(kt);
+   return ParMesh(MPI_COMM_WORLD, serial_mesh);
+}
+
+ParMesh CreateKershawMesh(int n, real_t eps)
+{
+   return CreateKershawMesh(n, n, n, eps, eps);
+}
+
+}
+
+#endif

miniapps/benchmarks/ceed-solver-bps/makefile

@@ -0,0 +1,77 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+MFEM_INSTALL_DIR ?= ../../../mfem
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/miniapps/benchmarks/ceed-solver-bps/,)
+CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
+   $(wildcard $(MFEM_INSTALL_DIR)/share/mfem/config.mk))
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_MINIAPPS =
+PAR_MINIAPPS = solver-bp
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+EXTRA_SOURCES = preconditioners.cpp
+EXTRA_HEADERS = kershaw.hpp rhs.hpp preconditioners.hpp
+EXTRA_OBJECTS = $(EXTRA_SOURCES:.cpp=.o)
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+.PRECIOUS: %.o
+
+# Remove built-in rules
+%: %.cpp
+%.o: %.cpp
+
+all: $(MINIAPPS)
+
+# Rule for building solver-bp
+solver-bp: solver-bp.o $(addprefix $(SRC),$(EXTRA_HEADERS)) \
+ $(EXTRA_OBJECTS) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_LINK_FLAGS) $< -o $@ $(EXTRA_OBJECTS) $(MFEM_LIBS)
+
+# Rules for compiling *.o files
+# -I$(MFEM_DIR) is needed for "general/forall.hpp" for out-of-source builds
+%.o: $(SRC)%.cpp $(wildcard $(SRC)%.hpp) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -I$(MFEM_DIR) -c $(<) -o $(@)
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Testing: Specific execution options
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+solver-bp-test-par: solver-bp
+	@$(call mfem-test,$<, $(RUN_MPI), CEED Solver BP,,SKIP-NO-VIS)
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_MINIAPPS) $(PAR_MINIAPPS) $(EXTRA_OBJECTS)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@true

miniapps/benchmarks/ceed-solver-bps/plot_csv.py

@@ -0,0 +1,129 @@
+import csv
+from pylab import *
+
+fields=[
+    ['code ID', 'str'],
+    ['preconditioner ID', 'str'],
+    ['machine ID', 'str'],
+    ['number of nodes', 'int'],
+    ['number of MPI ranks', 'int'],
+    ['n_x', 'int'], ['n_y', 'int'], ['n_z', 'int'],
+    ['solution polynomial degree', 'int'],
+    ['number of 1D quadrature points', 'float'],
+    ['eps_y', 'float'], ['eps_z', 'float'],
+    ['ndofs (including Dirichlet boundary)', 'int'],
+    ['niter', 'int'],
+    ['initial residual', 'float'], ['final residual', 'float'],
+    ['error', 'float'],
+    ['t_setup (preconditioner setup)', 'float'],
+    ['t_solve (total iter time)', 'float']]
+fields_dict=dict(fields)
+
+def convert(obj, type_str):
+    ctor=getattr(__builtins__, type_str)
+    return ctor(obj)
+
+input_csv='run-001.csv'
+print('reading %s ...' % input_csv)
+runs = []
+with open(input_csv) as csvfile:
+    csvreader = csv.DictReader(csvfile, fieldnames=[f[0] for f in fields],
+                               restkey='additional notes')
+    for row in csvreader:
+        for i in fields_dict:
+            row[i]=convert(row[i], fields_dict[i])
+        runs.append(row)
+
+orders=[r['solution polynomial degree'] for r in runs]
+orders=unique(orders) # numpy function
+# orders=[1]
+
+nps=[r['number of MPI ranks'] for r in runs]
+nps=unique(nps)
+if len(nps) > 1:
+    print('multiple num-ranks present: %s' % nps)
+    quit()
+np=nps[0]
+
+# plot fx (or fx/fn) vs fy, (or fx/fn/fy, etc) for all orders
+fn='number of MPI ranks'
+fx='ndofs (including Dirichlet boundary)'
+fy='t_solve (total iter time)'
+# fy='niter'
+# fy='error'
+fz='niter'
+
+figure()
+for p in orders:
+    rr=[r for r in runs if (r['solution polynomial degree']==p and
+                            r['niter']>0)]
+    if len(rr)==0:
+        continue
+
+    # pl_data=asarray([[r[fx],r[fx]/r[fy]] for r in rr])
+    # pl_data=asarray([[r[fx],r[fy]] for r in rr])
+    # pl_data=asarray([[r[fx],r[fx]/(r[fy]/r[fz])] for r in rr])
+
+    pl_data=asarray([[r[fx]/r[fn],r[fx]/r[fn]/r[fy]] for r in rr])
+    # pl_data=asarray([[r[fx]/r[fn],r[fy]] for r in rr])
+
+    plot(pl_data[:,0],pl_data[:,1], 'o-', label='p=%i'%p)
+    rnx=asarray([r['n_x'] for r in rr])
+    rerr=asarray([r['error'] for r in rr])
+    rate=arange(1.0,len(rnx))
+    for l in range(1,len(rnx)):
+        rate[l-1]=log(rerr[l-1]/rerr[l])/log(rnx[l]/rnx[l-1])
+    set_printoptions(formatter={'float':"{:6.2f}".format},linewidth=120)
+    print(f"p={p} rate:{rate}")
+
+# xscale('log', basex=10) # older matplotlib
+xscale('log', base=10)
+# xlim(4e4,3.1e7)
+xlim(4e4,5e6)
+# yscale('log', basey=10) # older matplotlib
+# yscale('log', base=10)
+# ylim(1e5,2e7)
+# ylim(0,2.55e7)
+# ylim(0,3.25e7)
+# ylim(0,5e6)
+ymin,ymax=ylim()
+ylim(0,ymax)
+# ylim(1e-2,2e1)
+# ylim(3e-3,6e-2)
+# xlabel(fx)
+# xlabel('# DOFs')
+xlabel('# DOFs / # Ranks')
+# ylabel(fx + ' / ' + fy)
+# ylabel(fy)
+# ylabel('# DOFs / t_solve')
+ylabel('# DOFs / # Ranks / t_solve')
+# ylabel('t_solve')
+# ylabel('# DOFs / (t_solve / # Iter)')
+# ylabel('# Iter')
+# ylabel('L2 error')
+# ylabel('Grad L2 error')
+grid('on', color='gray', ls='dotted')
+grid('on', axis='both', which='minor', color='gray', ls='dotted')
+legend(ncol=2, loc='best')
+ranks='1 MPI rank'
+if np > 1:
+   ranks='%s MPI ranks' % (np,np)
+hypre='hypre CPU'
+# hypre='hypre HIP'
+# prec=hypre+', p-MG(1,1)'
+prec=hypre+', LOR'
+# prec='Jacobi'
+# eps='1'
+eps='0.3'
+mfem='MFEM CPU'
+# mfem='MFEM HIP'
+title(mfem + ', ' + prec + ', $\\varepsilon = ' + eps + '$, ' + ranks)
+
+if 1: # write .pdf file?
+    pdf_file='plot.pdf'
+    print('saving figure --> %s'%pdf_file)
+    savefig(pdf_file, format='pdf', bbox_inches='tight')
+
+if 0: # show the figures?
+    print('\nshowing figures ...')
+    show()

miniapps/benchmarks/ceed-solver-bps/preconditioners.cpp

@@ -0,0 +1,241 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "preconditioners.hpp"
+
+namespace mfem
+{
+
+AssemblyLevel GetCoarseAssemblyLevel(SolverConfig config)
+{
+   switch (config.type)
+   {
+      case SolverConfig::JACOBI:
+      case SolverConfig::LOR_HYPRE:
+      case SolverConfig::LOR_AMGX:
+         return AssemblyLevel::PARTIAL;
+      default:
+         return AssemblyLevel::FULL;
+         // return AssemblyLevel::LEGACYFULL;
+   }
+}
+
+bool NeedsLOR(SolverConfig config)
+{
+   switch (config.type)
+   {
+      case SolverConfig::LOR_HYPRE:
+      case SolverConfig::LOR_AMGX:
+         return true;
+      default:
+         return false;
+   }
+}
+
+DiffusionMultigrid::DiffusionMultigrid(
+   ParFiniteElementSpaceHierarchy& hierarchy,
+   Coefficient &coeff_,
+   Array<int>& ess_bdr,
+   SolverConfig coarse_solver_config,
+   int q1d_inc_,
+   int smoothers_cheby_order_)
+   : GeometricMultigrid(hierarchy, ess_bdr),
+     coeff(coeff_),
+     q1d_inc(q1d_inc_),
+     irs(0, Quadrature1D::GaussLegendre),
+     smoothers_cheby_order(smoothers_cheby_order_)
+{
+   ConstructCoarseOperatorAndSolver(
+      coarse_solver_config, hierarchy.GetFESpaceAtLevel(0), ess_bdr);
+   int nlevels = hierarchy.GetNumLevels();
+   for (int i=1; i<nlevels; ++i)
+   {
+      ConstructOperatorAndSmoother(hierarchy.GetFESpaceAtLevel(i), ess_bdr);
+   }
+}
+
+void DiffusionMultigrid::ConstructBilinearForm(
+   ParFiniteElementSpace &fespace, Array<int> &ess_bdr, AssemblyLevel asm_lvl)
+{
+   ParBilinearForm *form = new ParBilinearForm(&fespace);
+   form->SetAssemblyLevel(asm_lvl);
+
+   DiffusionIntegrator *integ = new DiffusionIntegrator(coeff);
+
+   int p = fespace.GetOrder(0);
+   int dim = fespace.GetMesh()->Dimension();
+   // Integration rule for high-order problem: (p+1+q1d_inc)^d Gauss-Legendre
+   // points
+   int int_order = 2*(p+1+q1d_inc) - 1;
+   Geometry::Type geom = fespace.GetMesh()->GetElementBaseGeometry(0);
+   const IntegrationRule &ir = irs.Get(geom, int_order);
+   MFEM_VERIFY(ir.Size() == pow(p+1+q1d_inc,dim), "Wrong quadrature");
+   integ->SetIntegrationRule(ir);
+
+   form->AddDomainIntegrator(integ);
+   form->Assemble();
+   bfs.Append(form);
+
+   essentialTrueDofs.Append(new Array<int>());
+   fespace.GetEssentialTrueDofs(ess_bdr, *essentialTrueDofs.Last());
+}
+
+void DiffusionMultigrid::ConstructOperatorAndSmoother(
+   ParFiniteElementSpace& fespace, Array<int>& ess_bdr)
+{
+   ConstructBilinearForm(fespace, ess_bdr, AssemblyLevel::PARTIAL);
+
+   OperatorPtr opr;
+   bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
+   opr.SetOperatorOwner(false);
+
+   Vector diag(fespace.GetTrueVSize());
+   bfs.Last()->AssembleDiagonal(diag);
+
+   Solver* smoother = new OperatorChebyshevSmoother(
+      *opr, diag, *essentialTrueDofs.Last(), smoothers_cheby_order,
+      fespace.GetParMesh()->GetComm());
+
+   AddLevel(opr.Ptr(), smoother, true, true);
+}
+
+void DiffusionMultigrid::ConstructCoarseOperatorAndSolver(
+   SolverConfig config, ParFiniteElementSpace& fespace, Array<int>& ess_bdr)
+{
+   ConstructBilinearForm(fespace, ess_bdr, GetCoarseAssemblyLevel(config));
+   ParBilinearForm &a = static_cast<ParBilinearForm&>(*bfs.Last());
+   Array<int> &ess_dofs = *essentialTrueDofs.Last();
+
+   a.FormSystemMatrix(ess_dofs, A_coarse);
+
+   OperatorPtr A_prec;
+   if (NeedsLOR(config))
+   {
+      if (Mpi::Root())
+      {
+         std::cout << "Forming LOR discretization..." << std::endl;
+      }
+      lor.reset(new ParLORDiscretization(a, ess_dofs));
+      A_prec = lor->GetAssembledSystem();
+      if (Mpi::Root())
+      {
+         std::cout << "Forming LOR discretization... Done." << std::endl;
+      }
+   }
+   else
+   {
+      A_prec = A_coarse;
+   }
+
+   if (Mpi::Root()) { std::cout << "Forming preconditioner... " << std::endl; }
+   switch (config.type)
+   {
+      case SolverConfig::JACOBI:
+         coarse_precond.reset(new OperatorJacobiSmoother(a, ess_dofs));
+         break;
+      case SolverConfig::FA_HYPRE:
+      case SolverConfig::LOR_HYPRE:
+      {
+         HypreBoomerAMG *amg = new HypreBoomerAMG(*A_prec.As<HypreParMatrix>());
+         amg->SetPrintLevel(1);
+         Vector b(amg->Height());
+         Vector x(amg->Height());
+         b = 0.0;
+         x = 0.0;
+         amg->Setup(b, x); // Force setup;
+         coarse_precond.reset(amg);
+         break;
+      }
+#ifdef MFEM_USE_AMGX
+      case SolverConfig::FA_AMGX:
+      case SolverConfig::LOR_AMGX:
+      {
+         AmgXSolver *amg = new AmgXSolver;
+         amg->ReadParameters(config.amgx_config_file, AmgXSolver::EXTERNAL);
+         amg->InitExclusiveGPU(MPI_COMM_WORLD);
+         amg->SetOperator(*A_prec.As<HypreParMatrix>());
+         coarse_precond.reset(amg);
+         break;
+      }
+#endif
+      default:
+         MFEM_ABORT("Not available.")
+   }
+
+   if (config.inner_sli) // coarse_solver = SLI
+   {
+      SLISolver *sli = new SLISolver(fespace.GetComm());
+      sli->SetPrintLevel(0);
+      sli->SetAbsTol(0.0);
+      sli->SetRelTol(0.0);
+      sli->SetMaxIter(config.inner_sli_iter);
+      sli->SetOperator(*A_coarse);
+      sli->SetPreconditioner(*coarse_precond);
+      coarse_solver.reset(sli);
+   }
+   else if (config.inner_cg)
+   {
+      CGSolver *cg = new CGSolver(MPI_COMM_WORLD);
+      cg->SetPrintLevel(2);
+      cg->SetMaxIter(100);
+      cg->SetRelTol(1e-8);
+      cg->SetAbsTol(0.0);
+      cg->SetOperator(*A_coarse);
+      cg->SetPreconditioner(*coarse_precond);
+      cg->iterative_mode = false;
+      coarse_solver.reset(cg);
+   }
+   else
+   {
+      coarse_solver = coarse_precond;
+   }
+   if (Mpi::Root())
+   {
+      std::cout << "Forming preconditioner... Done.\n" << std::endl;
+   }
+
+   if (config.coarse_smooth)
+   {
+      Vector diag(fespace.GetTrueVSize());
+      a.AssembleDiagonal(diag);
+
+      Solver *smoother = new OperatorChebyshevSmoother(
+         *A_coarse, diag, ess_dofs, smoothers_cheby_order,
+         fespace.GetParMesh()->GetComm());
+
+      AddLevel(A_coarse.Ptr(), smoother, false, true);
+      AddCoarseSolver(coarse_solver.get(), false);
+   }
+   else
+   {
+      AddLevel(A_coarse.Ptr(), coarse_solver.get(), false, false);
+   }
+}
+
+void DiffusionMultigrid::SetSmoothersChebyshevOrder(int new_cheby_order)
+{
+   for (int level = MultigridBase::coarse_solver ? 0 : 1;
+        level < NumLevels(); level++)
+   {
+      OperatorChebyshevSmoother *cheby =
+         dynamic_cast<OperatorChebyshevSmoother*>(GetSmootherAtLevel(level));
+      if (cheby) { cheby->SetOrder(new_cheby_order); }
+   }
+   smoothers_cheby_order = new_cheby_order;
+}
+
+void DiffusionMultigrid::SetInnerSLINumIter(int inner_sli_iter)
+{
+   SLISolver *sli = dynamic_cast<SLISolver*>(coarse_solver.get());
+   if (sli) { sli->SetMaxIter(inner_sli_iter); }
+}
+
+} // namespace mfem

miniapps/benchmarks/ceed-solver-bps/preconditioners.hpp

@@ -0,0 +1,100 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef __SOLVER_BP_HPP__
+#define __SOLVER_BP_HPP__
+
+#include "mfem.hpp"
+#include <memory>
+
+namespace mfem
+{
+
+struct SolverConfig
+{
+   enum SolverType
+   {
+      JACOBI = 0,
+      FA_HYPRE = 1,
+      LOR_HYPRE = 2,
+      FA_AMGX = 3,
+      LOR_AMGX = 4
+   };
+   SolverType type;
+   const char *amgx_config_file = "amgx/amgx.json";
+   bool inner_cg = false; //<-- use inner CG iteration for coarse solver
+   bool inner_sli = false; //<-- use inner SLI iteration for coarse solver
+   int inner_sli_iter = 1; //<- number of iterations for the inner SLI solver
+   bool coarse_smooth = false; //<- enable level 0 smoothing
+   SolverConfig(SolverType type_) : type(type_) { }
+
+   void Print()
+   {
+      mfem::out << "Coarse solver: ";
+      switch (type)
+      {
+         case JACOBI: mfem::out << "Jacobi"; break;
+         case FA_HYPRE: mfem::out << "Hypre (full)"; break;
+         case LOR_HYPRE: mfem::out << "Hypre (LOR)"; break;
+         case FA_AMGX: mfem::out << "AmgX (full)"; break;
+         case LOR_AMGX: mfem::out << "AmgX (LOR)"; break;
+      }
+      mfem::out << std::endl;
+      // If inner_sli is true inner_cg is not used, see
+      // DiffusionMultigrid::ConstructCoarseOperatorAndSolver():
+      if (inner_sli) { inner_cg = false; }
+      mfem::out << "Inner CG:      "
+                << (inner_cg ? "On" : "Off")
+                << std::endl;
+      mfem::out << "Inner SLI:     " << (inner_sli ? "On" : "Off") << '\n';
+      mfem::out << "Coarse smooth: " << (coarse_smooth ? "On" : "Off") << '\n';
+   }
+};
+
+struct DiffusionMultigrid : GeometricMultigrid
+{
+   Coefficient &coeff;
+   int q1d_inc;
+   IntegrationRules irs;
+   std::unique_ptr<ParLORDiscretization> lor;
+   OperatorPtr A_coarse;
+   std::shared_ptr<Solver> coarse_solver, coarse_precond;
+   int smoothers_cheby_order;
+
+   DiffusionMultigrid(
+      ParFiniteElementSpaceHierarchy& hierarchy,
+      Coefficient &coeff_,
+      Array<int>& ess_bdr,
+      SolverConfig coarse_solver_config,
+      int q1d_inc_ = 0,
+      int smoothers_cheby_order_ = 1);
+
+   void ConstructBilinearForm(
+      ParFiniteElementSpace &fespace,
+      Array<int> &ess_bdr,
+      AssemblyLevel asm_lvl);
+
+   void ConstructOperatorAndSmoother(
+      ParFiniteElementSpace &fespace,
+      Array<int> &ess_bdr);
+
+   void ConstructCoarseOperatorAndSolver(
+      SolverConfig config,
+      ParFiniteElementSpace &fespace,
+      Array<int> &ess_bdr);
+
+   void SetSmoothersChebyshevOrder(int new_cheby_order);
+   void SetInnerSLINumIter(int inner_sli_iter);
+};
+
+} // namespace mfem
+
+#endif

miniapps/benchmarks/ceed-solver-bps/rhs.hpp

@@ -0,0 +1,334 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef __RHS_HPP__
+#define __RHS_HPP__
+
+#include "mfem.hpp"
+#include "general/forall.hpp"
+
+// 0 - Solution described in the CEED MS 36 report
+// 1 - Solution from the "ecp_special_2023" paper (option with cosine):
+//       w(n,x) = \sum_{k=0}^n a^k \cos(b^k \pi (x - 1/2)),   x \in [0,1]
+//     with a = 1/2, b = 3.
+// 2 - Solution from the "ecp_special_2023" paper (option with sine):
+//       w(n,x) = \sum_{k=0}^n a^k \sin(b^k \pi x),   x \in [0,1]
+//     with a = 1/2, b = 3.
+#define CEED_SOLVER_BP_SOLUTION_OPTION 1
+
+namespace mfem
+{
+
+constexpr real_t pi = real_t(M_PI);
+
+#if (CEED_SOLVER_BP_SOLUTION_OPTION == 0)
+
+MFEM_HOST_DEVICE inline
+real_t s(int k, real_t x)
+{
+   return sin(2*pi*k*x);
+}
+
+MFEM_HOST_DEVICE inline
+real_t u(int k, real_t x)
+{
+   real_t skx = s(k,x);
+   real_t sgn = skx < 0 ? -1.0 : 1.0;
+   return exp(-1/skx/skx)*sgn;
+}
+
+MFEM_HOST_DEVICE inline
+real_t u_xx(int k, real_t x)
+{
+   real_t kpix = k*pi*x;
+   real_t csc_2kpix = 1.0/sin(2*kpix);
+   real_t sgn = sin(2*kpix) < 0 ? -1.0 : 1.0;
+   return 2*exp(-csc_2kpix*csc_2kpix)*k*k*pi*pi
+          *(1 + 6*cos(4*kpix) + cos(8*kpix))
+          *pow(csc_2kpix,6)
+          *sgn;
+}
+
+MFEM_HOST_DEVICE inline
+real_t w(int n, real_t x)
+{
+   real_t wkx = 0.0;
+   real_t xx = 2*x - 1; // transform from [0,1] to [-1,1]
+   for (int j=0; j<n; ++j)
+   {
+      int k = pow(3, j);
+      wkx += u(k, xx);
+   }
+   return wkx;
+}
+
+MFEM_HOST_DEVICE inline
+real_t w_xx(int n, real_t x)
+{
+   real_t wkx = 0.0;
+   real_t xx = 2*x - 1; // transform from [0,1] to [-1,1]
+   if (xx == 0.0) { return 0.0; }
+   for (int j=0; j<n; ++j)
+   {
+      int k = pow(3, j);
+      wkx += 4*u_xx(k, xx); // factor of four from reference interval transf.
+   }
+   return wkx;
+}
+
+#elif (CEED_SOLVER_BP_SOLUTION_OPTION == 1)
+
+MFEM_HOST_DEVICE inline
+real_t w(int n, real_t x)
+{
+   //  w(n,x) = \sum_{k=0}^n a^k \cos(b^k \pi (x - 1/2))
+   const real_t a = 0.5, b = 3.;
+   real_t ak = 1.0;
+   real_t xk = pi * (x - 0.5);
+   real_t w_ = ak * cos(xk);
+   for (int k = 1; k <= n; k++)
+   {
+      ak *= a;
+      xk *= b;
+      w_ += ak * cos(xk);
+   }
+   return w_;
+}
+
+MFEM_HOST_DEVICE inline
+real_t w_x(int n, real_t x)
+{
+   //  w'(n,x) = -\pi \sum_{k=0}^n a^k b^k \sin(b^k \pi (x - 1/2))
+   const real_t a = 0.5, b = 3.;
+   real_t ck = -pi;
+   real_t xk = pi * (x - 0.5);
+   real_t w_x_ = ck * sin(xk);
+   for (int k = 1; k <= n; k++)
+   {
+      ck *= a * b;
+      xk *= b;
+      w_x_ += ck * sin(xk);
+   }
+   return w_x_;
+}
+
+MFEM_HOST_DEVICE inline
+real_t w_xx(int n, real_t x)
+{
+   // w''(n,x) = -\pi^2 \sum_{k=0}^n a^k b^{2 k} \cos(b^k \pi (x - 1/2))
+   const real_t a = 0.5, b = 3.;
+   real_t ck = -(pi * pi);
+   real_t xk = pi * (x - 0.5);
+   real_t w_xx_ = ck * cos(xk);
+   for (int k = 1; k <= n; k++)
+   {
+      ck *= a * b*b;
+      xk *= b;
+      w_xx_ += ck * cos(xk);
+   }
+   return w_xx_;
+}
+
+#elif (CEED_SOLVER_BP_SOLUTION_OPTION == 2)
+
+MFEM_HOST_DEVICE inline
+real_t w(int n, real_t x)
+{
+   //  w(n,x) = \sum_{k=0}^n a^k \sin(b^k \pi x)
+   const real_t a = 0.5, b = 3.;
+   real_t ak = 1.0;
+   real_t xk = pi * x;
+   real_t w_ = ak * sin(xk);
+   for (int k = 1; k <= n; k++)
+   {
+      ak *= a;
+      xk *= b;
+      w_ += ak * sin(xk);
+   }
+   return w_;
+}
+
+MFEM_HOST_DEVICE inline
+real_t w_xx(int n, real_t x)
+{
+   // w''(n,x) = -\pi^2 \sum_{k=0}^n a^k b^{2 k} \sin(b^k \pi x)
+   const real_t a = 0.5, b = 3.;
+   real_t ck = -(pi * pi);
+   real_t xk = pi * x;
+   real_t w_xx_ = ck * sin(xk);
+   for (int k = 1; k <= n; k++)
+   {
+      ck *= a * b*b;
+      xk *= b;
+      w_xx_ += ck * sin(xk);
+   }
+   return w_xx_;
+}
+
+#endif // CEED_SOLVER_BP_SOLUTION_OPTION
+
+struct ExactSolution : Coefficient
+{
+   int dim, n;
+   ExactSolution(int dim_, int n_=0) : dim(dim_), n(n_) { }
+   using Coefficient::Eval;
+   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override
+   {
+      real_t xyz[3];
+      Vector transip(xyz, 3);
+      T.Transform(ip, transip);
+      if (dim == 1)
+      {
+         return w(n, xyz[0]);
+      }
+      if (dim == 2)
+      {
+         return w(n, xyz[0])*w(n, xyz[1]);
+      }
+      else // dim == 3
+      {
+         return w(n, xyz[0])*w(n, xyz[1])*w(n, xyz[2]);
+      }
+   }
+};
+
+struct ExactGrad : VectorCoefficient
+{
+   int dim, n;
+   ExactGrad(int dim_, int n_)
+      : VectorCoefficient(dim_), dim(dim_), n(n_) { }
+   using VectorCoefficient::Eval;
+   void Eval(Vector &V, ElementTransformation &T,
+             const IntegrationPoint &ip) override
+   {
+      real_t xyz[3];
+      Vector transip(xyz, 3);
+      T.Transform(ip, transip);
+      V.SetSize(dim);
+      if (dim == 1)
+      {
+         V(0) = w_x(n, xyz[0]);
+      }
+      if (dim == 2)
+      {
+         V(0) = w_x(n, xyz[0])*  w(n, xyz[1]);
+         V(1) =   w(n, xyz[0])*w_x(n, xyz[1]);
+      }
+      else // dim == 3
+      {
+         const real_t wnx = w(n, xyz[0]);
+         const real_t wny = w(n, xyz[1]);
+         const real_t wnz = w(n, xyz[2]);
+         V(0) = w_x(n, xyz[0])*wny           *wnz;
+         V(1) = wnx           *w_x(n, xyz[1])*wnz;
+         V(2) = wnx           *wny           *w_x(n, xyz[2]);
+      }
+   }
+};
+
+MFEM_HOST_DEVICE inline
+real_t rhs_1d(const int n, const real_t *xyz)
+{
+   return -w_xx(n, xyz[0]);
+}
+
+MFEM_HOST_DEVICE inline
+real_t rhs_2d(const int n, const real_t *xyz)
+{
+   return -w_xx(n, xyz[0])*w(n, xyz[1]) - w(n, xyz[0])*w_xx(n, xyz[1]);
+}
+
+MFEM_HOST_DEVICE inline
+real_t rhs_3d(const int n, const real_t *xyz)
+{
+   return -w_xx(n, xyz[0])*w(n, xyz[1])*w(n, xyz[2])
+          - w(n, xyz[0])*w_xx(n, xyz[1])*w(n, xyz[2])
+          - w(n, xyz[0])*w(n, xyz[1])*w_xx(n, xyz[2]);
+}
+
+using RHSFunctionType = real_t(*)(int dim, const real_t *xyz);
+
+template <RHSFunctionType F>
+void ProjectRHS_(int n, QuadratureFunction &qf)
+{
+   QuadratureSpaceBase &qs = *qf.GetSpace();
+   Mesh &mesh = *qs.GetMesh();
+   const IntegrationRule &ir = qs.GetIntRule(0);
+
+   auto *geom = mesh.GetGeometricFactors(ir, GeometricFactors::COORDINATES);
+
+   const int dim = qs.GetMesh()->Dimension();
+   const int nq = ir.Size();
+   const int N = qf.Size();
+
+   const real_t *d_x = geom->X.Read();
+   real_t *d_q = qf.Write();
+
+   mfem::forall(N, [=] MFEM_HOST_DEVICE (int ii)
+   {
+      const int i = ii / nq;
+      const int j = ii % nq;
+      real_t xvec[3];
+      for (int d = 0; d < dim; ++d)
+      {
+         xvec[d] = d_x[j + d*nq + i*dim*nq];
+      }
+      d_q[ii] = F(n, xvec);
+   });
+}
+
+void ProjectRHS(int n, QuadratureFunction &qf)
+{
+   const int dim = qf.GetSpace()->GetMesh()->Dimension();
+   switch (dim)
+   {
+      case 1: ProjectRHS_<rhs_1d>(n, qf); break;
+      case 2: ProjectRHS_<rhs_2d>(n, qf); break;
+      case 3: ProjectRHS_<rhs_3d>(n, qf); break;
+      default: MFEM_ABORT("Unsupported dimension.");
+   }
+}
+
+struct RHS : Coefficient
+{
+   int dim, n;
+   RHS(int dim_, int n_=0) : dim(dim_), n(n_) { }
+   using Coefficient::Eval;
+   real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override
+   {
+      real_t xyz[3];
+      Vector transip(xyz, 3);
+      T.Transform(ip, transip);
+      if (dim == 1)
+      {
+         return -w_xx(n, xyz[0]);
+      }
+      if (dim == 2)
+      {
+         return -w_xx(n, xyz[0])*w(n, xyz[1]) - w(n, xyz[0])*w_xx(n, xyz[1]);
+      }
+      else // dim == 3
+      {
+         return -w_xx(n, xyz[0])*w(n, xyz[1])*w(n, xyz[2])
+                - w(n, xyz[0])*w_xx(n, xyz[1])*w(n, xyz[2])
+                - w(n, xyz[0])*w(n, xyz[1])*w_xx(n, xyz[2]);
+      }
+   }
+
+   void Project(QuadratureFunction &qf) override
+   {
+      ProjectRHS(n,qf);
+   }
+};
+
+}
+
+#endif

miniapps/benchmarks/ceed-solver-bps/run.sh

@@ -0,0 +1,124 @@
+bsep="============================================================"
+ssep="----------------------------------------"
+# Enable GPU-aware MPI:
+# gpu_aware_mpi_env_cmd="env MPICH_GPU_SUPPORT_ENABLED=1"
+# gpu_aware_mpi="-g"
+# number of nodes, number of MPI ranks:
+nnodes=1
+np=1
+# dev="-d hip ${gpu_aware_mpi}"
+eps="0.3"
+# mpirun_np="mpirun -np"
+mpirun_np="env MFEM_REPORT_KERNELS=1 mpirun -np"
+# mpirun_np="${gpu_aware_mpi_env_cmd} flux run --exclusive -N ${nnodes} -n"
+# dry run:
+# mpirun_np="echo ${mpirun_np}"
+#    p-MG/LOR + FA-hypre, or diagonal (Jacobi smoother)
+#    prec_type: "p-mg", "lor", or "diag"
+prec_type="lor"
+p_mg_opts="-cb 1"
+# p_mg_opts="-cb 5 -sli -sli-it 6"
+# lor_opts="-cls -cb 5 -sli -sli-it 6"
+# lor_opts="-cls -cb 2 -sli -sli-it 2"
+lor_opts="-cb 2 -sli -sli-it 2"
+mg_set=("1" "1 2" "1 3" "1 2 4" "1 3 5" "1 3 6")
+# mg_set=("1 2")
+#    p=7 and p=8 fail at the moment: "1 3 5 7" "1 3 5 8"
+# per-rank limits on the number of LOR elements for different p, in 2^20 units:
+# (bigger sizes run out of GPU memory, at least with LOR prec.)
+lor_ne_max_all=(4 4 4 4 4 4 4 4)
+# lor_ne_max_all=(18 22 24 24 27 24 8 8) # MI250X
+((lor_ne_min = 40*2**10))
+((np_ = np))
+((mm = 1))
+while ((np_ > 8)); do
+   ((mm++))
+   ((np_ = (np_-1)/8+1))
+done
+((mf = 2**mm))
+((mff = 3*mf))
+echo " *** mf = ${mf}, mff = ${mff}"
+for mg in "${mg_set[@]}"; do
+   echo "${bsep}"
+   p=(${mg})
+   # p=${p[-1]}
+   p="${p[$((${#p[@]}-1))]}"
+   lor_ne_max="${lor_ne_max_all[$((p-1))]}"
+   ((lor_ne_max *= 2**20))
+   # n_max = floor(lor_ne_max^(1/3))
+   n_max=$(echo "a=e((1/3)*l(${np}*${lor_ne_max}));scale=0;a/1" | bc -l)
+   # for np*lor_ne_max=256^3, the above gives 255, so we adjust the result:
+   while (( (n_max+1)**3 <= np*lor_ne_max )); do
+      ((n_max++))
+   done
+   echo " *** p = ${p}, n_max = ${n_max}"
+   if (( n_max**3 > np*lor_ne_max )); then
+      echo "error: n_max^3 > np*lor_ne_max"
+      exit 1
+   fi
+   echo "${bsep}"
+   nx_set=()
+   for ((nx = (n_max/p/mff)*mff, last_nx = 2*nx; nx >= 6; nx -= mff)); do
+      ((last_ne = last_nx**3))
+      ((ne = nx**3))
+      ((lor_ne = (p*nx)**3))
+      if ((np*lor_ne_min > lor_ne)); then break; fi
+      if ((last_ne < ne*4/3)); then continue; fi
+      nx_set=("${nx}" "${nx_set[@]}")
+      ((ndofs = (p*nx+1)**3))
+      ((rhs_n=0))
+      while ((2*3**(rhs_n+1) <= p*nx)); do
+         ((rhs_n++))
+      done
+      # 2*3**rhs_n <= p*nx < 2*3**(rhs_n+1)
+      printf "np = ${np}, p = ${p}, nx = ${nx}, ndofs = ${ndofs}"
+      # rhs_n for eps = 1:
+      # printf ", rhs_n = ${rhs_n}"
+      printf "\n"
+      ((last_nx = nx))
+   done
+   for nx in "${nx_set[@]}"; do
+      # break;
+      if ((nx % mf != 0)); then
+         echo " *** internal error!"
+         exit 1
+      fi
+      ((rp = mm))
+      ((nx /= mf))
+      if false; then
+         # 0, 1, or 2 additional parallel refinements for 1, 8, or 64 ranks
+         ((np_=np))
+         while ((np_%8 == 0)); do
+            ((np_=np_/8))
+            ((rp++))
+         done
+      fi
+      ((ndofs = (p*nx*2**rp+1)**3))
+      echo "${bsep}"
+      echo "np = ${np}, p = ${p}, ndofs = ${ndofs}"
+      if [[ "$prec_type" == "p-mg" ]]; then
+         # p-MG
+         printf "$mpirun_np ${np} ./solver-bp ${dev}"
+         printf " -ey ${eps} -mg \"${mg}\" -cs 1 ${p_mg_opts}"
+         printf " -nx ${nx} -rp ${rp}\n"
+         echo "${ssep}"
+         $mpirun_np "${np}" ./solver-bp ${dev} \
+            -ey ${eps} -mg "${mg}" -cs 1 ${p_mg_opts} -nx "${nx}" -rp "${rp}"
+      elif [[ "$prec_type" == "lor" ]]; then
+         # LOR
+         printf "$mpirun_np ${np} ./solver-bp ${dev}"
+         printf " -ey ${eps} -mg \"${p}\" -cs 2 ${lor_opts}"
+         printf " -nx ${nx} -rp ${rp}\n"
+         echo "${ssep}"
+         $mpirun_np "${np}" ./solver-bp ${dev} \
+            -ey ${eps} -mg "${p}" -cs 2 ${lor_opts} -nx "${nx}" -rp "${rp}"
+      elif [[ "$prec_type" == "diag" ]]; then
+         # Diag
+         printf "$mpirun_np ${np} ./solver-bp ${dev}"
+         printf " -ey ${eps} -mg \"${p}\" -cs 0 -nx ${nx} -rp ${rp}\n"
+         echo "${ssep}"
+         $mpirun_np "${np}" ./solver-bp ${dev} \
+            -ey ${eps} -mg "${p}" -cs 0 -nx "${nx}" -rp "${rp}"
+      fi
+   done
+done

miniapps/benchmarks/ceed-solver-bps/solver-bp.cpp

@@ -0,0 +1,811 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// --------------------------------------------------------------
+//    MFEM Implementation of the CEED Solver Bake-off Problems
+// --------------------------------------------------------------
+//
+// Run a suite of benchmarks and view the results:
+//
+//   1. Edit 'run.sh' to adjust machine and size parameters.
+//   2. Run 'run.sh' redirecting output to a file, e.g.:
+//        bash run.sh > run-001.out
+//   3. Extract the CSV output:
+//        sed -n -e 's/^= CSV:\(.*\)$/\1/p' run-001.out > run-001.csv
+//   3. Edit the script 'plot_csv.py' set the name of your CSV file and,
+//      optionally, customize the plot it generates.
+//   4. Process the CSV file:
+//        python3 plot_csv.py
+//
+// Sample runs:
+//
+//   solver-bp -nx 6
+//   solver-bp -nx 6 -mg "1 2 3"
+//   solver-bp -nx 6 -mg "1 r r 2 3"
+//   solver-bp -nx 6 -rp 2 -mg 3 -cs 1
+//   solver-bp -nx 6 -rp 2 -mg 3 -cs 2
+//
+// Device sample runs:
+//
+//   solver-bp -d cuda -nx 6 -mg "1 r r 2 3" -cs 0
+//   solver-bp -d cuda -nx 6 -rp 2 -mg 3 -cs 3
+//   solver-bp -d cuda -nx 6 -rp 2 -mg 3 -cs 4
+//
+
+#include "mfem.hpp"
+#include "kershaw.hpp"
+#include "rhs.hpp"
+#include "preconditioners.hpp"
+#include <regex>
+#include <fem/integ/bilininteg_diffusion_kernels.hpp>
+
+using namespace std;
+using namespace mfem;
+
+struct MGRefinement
+{
+   enum Type { P_MG, H_MG };
+   const Type type;
+   const int order;
+   MGRefinement(Type type_, int order_) : type(type_), order(order_) { }
+   static MGRefinement p(int order_) { return MGRefinement(P_MG, order_); }
+   static MGRefinement h() { return MGRefinement(H_MG, 0); }
+};
+
+struct CGMonitor : IterativeSolverMonitor
+{
+   const real_t tol;
+   real_t initial_nrm, final_nrm, saved_nrm;
+   int final_it, saved_it;
+
+   CGMonitor(real_t tol_) : tol(tol_) { }
+
+   void MonitorResidual(int it, real_t norm, const Vector &r, bool final)
+   override
+   {
+      MFEM_CONTRACT_VAR(norm);
+      // Avoid recomputing the norm if it was already computed -- this method
+      // is called two times for the final iteration: once with final = false
+      // (possibly triggering the monitor convergence criterion) and a second
+      // time with final = true.
+      bool init_call = (it == 0 && !final);
+      const real_t nrm =
+         (!init_call && it == saved_it) ?
+         saved_nrm :
+         sqrt(InnerProduct(iter_solver->GetComm(), r, r));
+      if ((it == 0 || final) && Mpi::Root())
+      {
+         mfem::out << (final ? "Final" : "   Initial")
+                   << " l2 norm of residual: " << nrm << '\n';
+      }
+      if (init_call)
+      {
+         initial_nrm = nrm;
+         converged = false;
+         final_nrm = -1.0;
+         final_it = -1;
+      }
+      saved_nrm = nrm;
+      saved_it = it;
+      // Check for monitor-triggered convergence
+      converged = (nrm <= tol*initial_nrm);
+      if (final)
+      {
+         final_nrm = nrm;
+         final_it = it;
+      }
+      if (final && Mpi::Root())
+      {
+         mfem::out << "Final relative l2 residual: ";
+         if (initial_nrm == 0.0)
+         {
+            mfem::out << "N/A (initial norm is 0)" << endl;
+         }
+         else
+         {
+            const real_t rel_nrm = nrm/initial_nrm;
+            mfem::out << rel_nrm << '\n';
+            mfem::out << "Average l2 reduction factor: ";
+            if (it == 0) { mfem::out << "N/A"; }
+            else { mfem::out << pow(rel_nrm, 1.0/it); }
+            mfem::out << " [" << it << " iterations]" << endl;
+         }
+      }
+   }
+};
+
+void report_hypre_gpu_status(bool gpu_aware_mpi_requested);
+void report_env_vars();
+real_t verify_ess_bdr(const Vector &b, const Vector &x,
+                      const Array<int> &ess_tdof_list);
+
+template <typename T> void PrintPair(const string &name, T val)
+{
+   cout << setw(14) << left << name << val << '\n';
+}
+
+
+int main(int argc, char *argv[])
+{
+   DiffusionIntegrator::AddSpecialization<3,3,3>();
+   DiffusionIntegrator::AddSpecialization<3,4,4>();
+   DiffusionIntegrator::AddSpecialization<3,5,5>();
+   DiffusionIntegrator::AddSpecialization<3,6,6>();
+
+   Mpi::Init(argc, argv);
+   Hypre::Init();
+
+   const char *device_config = "cpu";
+   bool gpu_aware_mpi = false;
+   int nx = 6, ny = -1, nz = -1;
+   int rhs_n = -1;
+   const char *mg_spec = "1";
+   int q1d_inc = 0; // num 1D qpts = p + 1 + q1d_inc
+   int smoothers_cheby_order = 1;
+   real_t epsy = 1.0, epsz = -1;
+   int ref_par = 0;
+   bool glvis = false;
+   bool paraview = false;
+   SolverConfig coarse_solver(SolverConfig::JACOBI);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&gpu_aware_mpi, "-g", "--gpu-aware-mpi", "-no-g",
+                  "--no-gpu-aware-mpi", "Enable GPU-aware MPI.");
+   args.AddOption(&mg_spec, "-mg", "--multigrid-spec",
+                  "Multigrid specification. See README for description.");
+   args.AddOption(&q1d_inc, "-qi", "--quadrature-points-increment",
+                  "Increment for the 1D quadrature points relative to p + 1");
+   args.AddOption(&smoothers_cheby_order, "-cb",
+                  "--smoothers-chebyshev-order",
+                  "Order of the Chebyshev smoothers for the multigrid.");
+   args.AddOption((int*)&coarse_solver.type, "-cs", "--coarse-solver-config",
+                  "Coarse solver configuration. 0: Jacobi, 1: FA-HYPRE, "
+                  "2: LOR-HYPRE, 3: FA-AMGX, 4: LOR-AMGX.");
+   args.AddOption(&coarse_solver.inner_cg, "-cg", "--inner-cg",
+                  "-no-cg", "--no-inner-cg",
+                  "Use inner CG iteration for the coarse solver.");
+   args.AddOption(&coarse_solver.inner_sli, "-sli", "--inner-sli",
+                  "-no-sli", "--no-inner-sli",
+                  "Use inner SLI iteration for the coarse solver.");
+   args.AddOption(&coarse_solver.inner_sli_iter, "-sli-it",
+                  "--inner-sli-iterations",
+                  "Number of iterations for the inner SLI solver.");
+   args.AddOption(&coarse_solver.coarse_smooth, "-cls", "--coarse-level-smooth",
+                  "-no-cls", "--no-coarse-level-smooth",
+                  "Use coarse smoothing in addition to the coarse solver.");
+   args.AddOption(&coarse_solver.amgx_config_file, "-amgx", "--amgx-config",
+                  "AmgX config JSON file.");
+   args.AddOption(&nx, "-nx", "--nx", "Number of elements in x direction.");
+   args.AddOption(&ny, "-ny", "--ny", "Number of elements in y direction.");
+   args.AddOption(&nz, "-nz", "--nz", "Number of elements in z direction.");
+   args.AddOption(&epsy, "-ey", "--epsy", "Kershaw parameter epsilon y.");
+   args.AddOption(&epsz, "-ez", "--epsz", "Kershaw parameter epsilon z.");
+   args.AddOption(&rhs_n, "-rn", "--rhs-n",
+                  "Parameter n in the RHS function; -1 for default.");
+   args.AddOption(&ref_par, "-rp", "--ref-par",
+                  "Number of uniform parallel refinements to perform.");
+   args.AddOption(&glvis, "-gv", "--glvis", "-no-gv", "--no-glvis",
+                  "Save the mesh and solution for GLVis visualization.");
+   args.AddOption(&paraview, "-pv", "--paraview", "-no-pv", "--no-paraview",
+                  "Save data files for ParaView visualization.");
+   args.ParseCheck();
+
+   if (ny < 0) { ny = nx; }
+   if (nz < 0) { nz = nx; }
+   if (epsz < 0) { epsz = epsy; }
+   // rhs_n default is handled later
+
+   Device device(device_config);
+   device.SetGPUAwareMPI(gpu_aware_mpi);
+   if (Mpi::Root()) { device.Print(); }
+   // Report HYPRE's GPU config and GPU-aware MPI config. Terminates if
+   // GPU-aware MPI is requested but HYPRE's GPU-aware MPI support is disabled.
+   report_hypre_gpu_status(gpu_aware_mpi);
+   // Report environment variables like {CUDA,ROCR}_VISIBLE_DEVICES:
+   report_env_vars();
+
+   // Generate mesh
+   ParMesh mesh_coarse = CreateKershawMesh(nx, ny, nz, epsy, epsz);
+   const int dim = mesh_coarse.Dimension();
+   for (int i=0; i<ref_par; ++i) { mesh_coarse.UniformRefinement(); }
+
+   int coarse_order = 0, order = 0, h_ref = ref_par;
+   // Parse order specification
+   vector<MGRefinement> mg_refinements;
+   {
+      istringstream mg_stream(mg_spec);
+      string ref;
+      mg_stream >> coarse_order;
+      int prev_order = order = coarse_order;
+      if (Mpi::Root()) { cout << "\nCoarse order " << coarse_order << '\n'; }
+      while (mg_stream >> ref)
+      {
+         if (ref == "r")
+         {
+            if (Mpi::Root()) { cout << "h-MG uniform refinement\n"; }
+            mg_refinements.push_back(MGRefinement::h());
+            ++h_ref;
+         }
+         else
+         {
+            try { order = stoi(ref); }
+            catch (...)
+            {
+               MFEM_ABORT("Multigrid refinement must either be an integer or "
+                          "the character `r`");
+            }
+            if (Mpi::Root()) { cout << "p-MG order   " << order << '\n'; }
+            MFEM_VERIFY(order > 0, "Orders must be positive");
+            MFEM_VERIFY(order > prev_order, "Orders must be increasing");
+            mg_refinements.push_back(MGRefinement::p(order));
+            prev_order = order;
+         }
+      }
+   }
+
+   if (order == 1 && coarse_solver.type == SolverConfig::LOR_HYPRE)
+   {
+      // Using ~10^7 elements with p=1 overflows a Vector in the LOR setup.
+      // The Vector has size (3D): (p+1)^3 * 27 * num_elem_ho.
+      // In 3D, for p > 1, the overflow will happen around:
+      // - p=2: ~23.6 million dofs or 2,945,794 elements
+      // - p=3: ~33.6 million dofs or 1,242,757 elements
+      // - p=4: ~40.7 million dofs or   636,292 elements
+      // - p=5: ~46.0 million dofs or   368,225 elements
+      // - p=6: ~50.1 million dofs or   231,885 elements
+      //
+      // Note: the size of the Jacobians at quadrature points (with q1d=p+1) in
+      // 3D is: (p+1)^3 * 9 * num_elem, so 3x smaller than the above Vector.
+      //
+      // For q1d=p+2, the overflow happens around:
+      // - p=1: 8,837,382 elements or ~8.8 million dofs
+      // - p=2: 3,728,271 elements or ~29.8 million dofs
+      // - p=3: 1,908,875 elements or ~51.5 million dofs
+      // - p=4: 1,104,673 elements or ~70.7 million dofs
+      // - p=5:   695,654 elements or ~87.0 million dofs
+      // - p=6:   466,034 elements or ~100.7 million dofs
+      coarse_solver.type = SolverConfig::FA_HYPRE;
+      if (Mpi::Root())
+      {
+         cout << "\nOrder is 1: switching from LOR-HYPRE to FA-HYPRE.\n";
+      }
+   }
+#if 0
+   if (order == 1 && coarse_solver.type == SolverConfig::FA_HYPRE &&
+       coarse_solver.inner_sli)
+   {
+      coarse_solver.inner_sli = false;
+      if (Mpi::Root())
+      {
+         cout << "\nOrder is 1: turning off the inner SLI.\n";
+      }
+   }
+#endif
+
+   vector<unique_ptr<FiniteElementCollection>> fe_collections;
+   fe_collections.emplace_back(new H1_FECollection(coarse_order, dim));
+   ParFiniteElementSpace fes_coarse(&mesh_coarse, fe_collections.back().get());
+   ParFiniteElementSpaceHierarchy hierarchy(&mesh_coarse, &fes_coarse,
+                                            false, false);
+
+   for (MGRefinement ref : mg_refinements)
+   {
+      if (ref.type == MGRefinement::H_MG)
+      {
+         hierarchy.AddUniformlyRefinedLevel();
+      }
+      else // P_MG
+      {
+         fe_collections.emplace_back(new H1_FECollection(ref.order, dim));
+         hierarchy.AddOrderRefinedLevel(fe_collections.back().get());
+      }
+   }
+
+   const int nlevels = hierarchy.GetNumLevels();
+   if (Mpi::Root())
+   {
+      if (nlevels == 1)
+      {
+         cout << "1 level in MG hierarchy. Using coarse solver only." << endl;
+      }
+      else
+      {
+         cout << nlevels << " levels in MG hierarchy." << endl;
+      }
+      coarse_solver.Print();
+      cout << endl;
+   }
+
+   // Determine final nx, ny, nz and use them to determine the default rhs_n.
+   const int ref_factor = pow(2, h_ref);
+   nx *= ref_factor;
+   ny *= ref_factor;
+   nz *= ref_factor;
+   if (rhs_n < 0)
+   {
+      int n_min = min(nx, ny);
+      if (nz > 0) { n_min = min(n_min, nz); }
+      // Find rhs_n such that 2*3^rhs_n <= (order*n_min) < 2*3^{rhs_n+1}
+      rhs_n = 0;
+      for (int l = 2*3; l <= order*n_min; l *= 3) { rhs_n++; }
+      if (epsy < 0.8) { rhs_n--; }
+      if (Mpi::Root()) { cout << "Using rhs_n = " << rhs_n << '\n' << endl; }
+   }
+
+   ParFiniteElementSpace &fes = hierarchy.GetFinestFESpace();
+   ParMesh &mesh = *fes.GetParMesh();
+   mesh.PrintInfo(cout);
+   HYPRE_Int ndof = fes.GlobalTrueVSize();
+   if (Mpi::Root())
+   {
+      cout << "\nTotal number of DOFs: " << ndof << endl << endl;
+   }
+
+   // All Dirichlet boundaries
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient coeff(1.0); // Diffusion coefficient
+   // Set up RHS
+   if (Mpi::Root()) { cout << "Assembling right-hand side..." << endl; }
+   RHS rhs_coeff(dim, rhs_n);
+   ParLinearForm b(&fes);
+   const int rhs_ir_inc = 2*q1d_inc+1;
+   // --> ir_order = 2*(p+1+q1d_inc)-1 --> q1d = p+1+q1d_inc
+   b.AddDomainIntegrator(new DomainLFIntegrator(rhs_coeff, 2, rhs_ir_inc));
+   b.UseFastAssembly(true);
+   b.Assemble();
+   if (Mpi::Root()) { cout << "Assembling right-hand side... Done." << endl; }
+
+   // make sure the GPU is done with any previous tasks:
+   if (Device::Allows(Backend::DEVICE_MASK)) { MFEM_STREAM_SYNC; }
+   // make sure all ranks are done with any previous tasks:
+   MPI_Barrier(MPI_COMM_WORLD);
+   tic();
+   // Set up operators in the multigrid hierarchy
+   DiffusionMultigrid MG(hierarchy, coeff, ess_bdr, coarse_solver, q1d_inc,
+                         smoothers_cheby_order);
+   MG.SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1);
+   // make sure the GPU is done with all setup tasks:
+   if (Device::Allows(Backend::DEVICE_MASK)) { MFEM_STREAM_SYNC; }
+   // make sure all ranks are done with all setup tasks:
+   MPI_Barrier(MPI_COMM_WORLD);
+   const real_t t_setup = tic_toc.RealTime();
+
+   ParGridFunction x(&fes);
+   x = 0.0;
+
+   OperatorPtr A;
+   Vector X, B;
+   MG.FormFineLinearSystem(x, b, A, X, B);
+
+   const real_t l2_tol = 1e-8;
+   CGMonitor monitor(l2_tol);
+
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(0.0); // use the 'monitor' for convergence
+   cg.SetPrintLevel(1);
+   cg.SetOperator(*A);
+   cg.SetPreconditioner(MG);
+   cg.SetMonitor(monitor);
+   // Run 2 CG iterations to ensure everything is allocated and initialized for
+   // the full CG solve:
+   if (Mpi::Root()) { cout << "Running 2 warm-up CG iterations ...\n"; }
+   cg.SetMaxIter(2);
+   {
+      Vector X_save(X);
+      cg.Mult(B, X);
+      X = X_save;
+   }
+   if (coarse_solver.inner_sli &&
+       ((coarse_solver.type == SolverConfig::FA_HYPRE /* && order > 1 */) ||
+        coarse_solver.type == SolverConfig::LOR_HYPRE))
+   {
+      // timing data: (t-solve,sli-iter,cheby-order,pcg-iter)
+      std::vector<std::tuple<double,int,int,int>> timings;
+      Vector X_save(X);
+      if (Mpi::Root()) { cout << "\nFinding optimal MG parameters ...\n"; }
+      cg.SetMaxIter(500);
+      for (int sli_it = 1; sli_it <= coarse_solver.inner_sli_iter; sli_it++)
+      {
+         MG.SetInnerSLINumIter(sli_it);
+         for (int cheby_order = 1; cheby_order <= smoothers_cheby_order;
+              cheby_order++)
+         {
+            MG.SetSmoothersChebyshevOrder(cheby_order);
+
+            if (Mpi::Root())
+            {
+               cout << "\nRunning and timing parameters (sli iter, cheby order)"
+                    << " = (" << sli_it << ',' << cheby_order << ") ...\n";
+            }
+            // make sure the GPU is done with any previous tasks:
+            if (Device::Allows(Backend::DEVICE_MASK)) { MFEM_STREAM_SYNC; }
+            // make sure all ranks are done with any previous tasks:
+            MPI_Barrier(MPI_COMM_WORLD);
+            tic();
+            cg.Mult(B, X);
+            // make sure the GPU is done with all solve tasks:
+            if (Device::Allows(Backend::DEVICE_MASK)) { MFEM_STREAM_SYNC; }
+            // make sure all ranks are done with all solve tasks:
+            MPI_Barrier(MPI_COMM_WORLD);
+            const double t_solve = tic_toc.RealTime();
+            if (cg.GetConverged())
+            {
+               timings.emplace_back(t_solve, sli_it, cheby_order,
+                                    cg.GetNumIterations());
+            }
+            X = X_save;
+         }
+      }
+      std::sort(timings.begin(), timings.end());
+      if (Mpi::Root())
+      {
+         cout << "\nSorted timings from rank 0:\n";
+         const auto old_prec = cout.precision(6);
+         const auto old_fmtflags = cout.flags();
+         cout << std::fixed;
+         for (size_t i = 0; i < timings.size(); i++)
+         {
+            cout << setw(2) << i << ": "
+                 << 1e3*std::get<0>(timings[i]) << " ms: ("
+                 << std::get<1>(timings[i]) << ','
+                 << std::get<2>(timings[i]) << "): "
+                 << setw(3) << std::get<3>(timings[i]) << " iter\n";
+         }
+         cout.flags(old_fmtflags);
+         cout.precision(old_prec);
+      }
+      if (timings.size() > 0)
+      {
+         // Use the fastest parameters (as timed on rank 0) for the full solve:
+         int si = std::get<1>(timings[0]);
+         int co = std::get<2>(timings[0]);
+         MPI_Bcast(&si, 1, MPI_INT, 0, MPI_COMM_WORLD);
+         MPI_Bcast(&co, 1, MPI_INT, 0, MPI_COMM_WORLD);
+         MG.SetInnerSLINumIter(si);
+         MG.SetSmoothersChebyshevOrder(co);
+         coarse_solver.inner_sli_iter = si;
+         smoothers_cheby_order = co;
+         if (Mpi::Root())
+         {
+            cout << "\nUsing the fastest option (sli iter, cheby order) = ("
+                 << si << ',' << co << ")\n";
+         }
+      }
+      else
+      {
+         MG.SetInnerSLINumIter(1);
+         MG.SetSmoothersChebyshevOrder(1);
+         coarse_solver.inner_sli_iter = 1;
+         smoothers_cheby_order = 1;
+         if (Mpi::Root())
+         {
+            cout << "\nAll options failed to converge!"
+                 << " Using (sli iter, cheby order) = (1,1)\n";
+         }
+      }
+   }
+   if (Mpi::Root()) { cout << "\nRunning and timing the full CG solve ...\n"; }
+   cg.SetMaxIter(500);
+   // make sure the GPU is done with any previous tasks:
+   if (Device::Allows(Backend::DEVICE_MASK)) { MFEM_STREAM_SYNC; }
+   // make sure all ranks are done with any previous tasks:
+   MPI_Barrier(MPI_COMM_WORLD);
+   tic();
+   cg.Mult(B, X);
+   // make sure the GPU is done with all solve tasks:
+   if (Device::Allows(Backend::DEVICE_MASK)) { MFEM_STREAM_SYNC; }
+   // make sure all ranks are done with all solve tasks:
+   MPI_Barrier(MPI_COMM_WORLD);
+   const double t_solve = tic_toc.RealTime();
+
+   const int niter = cg.GetConverged() ? cg.GetNumIterations() : -1;
+
+   const real_t bdr_err = verify_ess_bdr(B, X, MG.GetFineEssentialTrueDofs());
+   if (Mpi::Root())
+   {
+      MFEM_VERIFY(bdr_err == 0.0, "Incorrect boundary values in solution!"
+                  " bdr_err = " << bdr_err);
+   }
+
+   MG.RecoverFineFEMSolution(X, b, x);
+
+   ExactSolution exact_coeff(dim, rhs_n);
+   // ExactGrad exact_grad_coeff(dim, rhs_n);
+   real_t L2_err = x.ComputeL2Error(exact_coeff);
+   // real_t grad_err = x.ComputeGradError(&exact_grad_coeff);
+   if (Mpi::Root())
+   {
+      cout << "\nL2 Error: " << setprecision(10) << scientific
+           << L2_err << '\n';
+      // cout << "\nGrad Error: " << setprecision(10) << scientific
+      //      << grad_err << '\n';
+   }
+
+   if (glvis)
+   {
+      ofstream mesh_ofs(MakeParFilename("mesh.", Mpi::WorldRank()));
+      mesh_ofs.precision(8);
+      mesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(MakeParFilename("sol.", Mpi::WorldRank()));
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   if (paraview)
+   {
+      ParGridFunction rhs_gf(&fes), exact_gf(&fes), error_gf(&fes);
+      rhs_gf.ProjectCoefficient(rhs_coeff);
+      exact_gf.ProjectCoefficient(exact_coeff);
+
+      subtract(exact_gf, x, error_gf);
+
+      ParaViewDataCollection dc("SolverBP", &mesh);
+      dc.RegisterField("u", &x);
+      dc.RegisterField("rhs", &rhs_gf);
+      dc.RegisterField("exact", &exact_gf);
+      dc.RegisterField("error", &error_gf);
+      dc.SetPrefixPath("ParaView");
+      dc.SetLevelsOfDetail(order);
+      dc.SetHighOrderOutput(true);
+      dc.SetCycle(0);
+      dc.SetTime(0.0);
+      dc.Save();
+   }
+
+   const long long nel = mesh.GetGlobalNE();
+   if (nz == 0) { MFEM_VERIFY(nel == nx*ny, "Wrong number of elements"); }
+   else { MFEM_VERIFY(nel == nx*ny*nz, "Wrong number of elements"); }
+
+   if (Mpi::Root())
+   {
+      cout << "\n= Results\n";
+      PrintPair("nranks", Mpi::WorldSize());
+      PrintPair("nx", nx);
+      PrintPair("ny", ny);
+      PrintPair("nz", nz);
+      PrintPair("degree", order);
+      PrintPair("rhs_n", rhs_n);
+      PrintPair("epsy", epsy);
+      PrintPair("epsz", epsz);
+      PrintPair("ndof", ndof);
+      PrintPair("niter", niter);
+
+      // Should also output:
+      // code id
+      // prec id
+      // machine id
+      // number of supercomputer nodes
+      // number of 1d quadrature points
+      // initial and final residuals
+      // error
+
+      // Timings
+      PrintPair("t_setup", t_setup);
+      PrintPair("t_solve", t_solve);
+
+      cout << "\nSolve MDOFs/rank/sec:   "
+           << ndof/1e6/Mpi::WorldSize()/t_solve << '\n';
+
+      // CSV fields:
+      // 1. code ID
+      // 2. preconditioner ID
+      // 3. machine ID
+      // 4. number of nodes
+      // 5. number of MPI ranks
+      // 6,7,8. n_x, n_y, n_z
+      // 9. solution polynomial degree
+      // 10. number of 1D quadrature points
+      // 11,12. eps_y, eps_z
+      // 13. ndofs (including Dirichlet boundary)
+      // 14. niter
+      // 15,16. initial and final residuals
+      // 17. error
+      // 18. t_setup (preconditioner setup)
+      // 19. t_solve (total iter time)
+      //
+      // extract the CSV lines from the output with:
+      //    grep "= CSV:" out.txt | sed -e 's/^= CSV://' > out.csv
+      cout << "\n= CSV:"
+           << "MFEM-" + string(device_config); // 1
+      string hypre_str =
+#if defined(HYPRE_USING_HIP)
+         "hypre-hip"
+#elif defined(HYPRE_USING_CUDA)
+         "hypre-cuda"
+#else
+         "hypre-cpu"
+#endif
+         ;
+      auto cs = coarse_solver.type;
+      string prec_id;
+      if (cs == SolverConfig::FA_HYPRE) // p-MG, add (sli-iter,cheby-order)
+      {
+         prec_id = hypre_str + "-pMG(";
+      }
+      else if (cs == SolverConfig::LOR_HYPRE) // LOR, add (sli-iter,cheby-order)
+      {
+         prec_id = hypre_str + "-LOR(";
+      }
+      else if (cs == SolverConfig::JACOBI)
+      {
+         prec_id = "diag(";
+      }
+      else
+      {
+         prec_id = "(unknown)(";
+      }
+      if (coarse_solver.inner_cg)
+      {
+         prec_id += "cg;";
+      }
+      if (coarse_solver.inner_sli)
+      {
+         prec_id += to_string(coarse_solver.inner_sli_iter) + ";";
+      }
+      prec_id += to_string(smoothers_cheby_order) +
+                 (coarse_solver.coarse_smooth ? "c" : "") + ")";
+      prec_id += "-" + regex_replace(mg_spec, regex(" "), "-");
+      cout << ',' << prec_id; // 2
+      const char *hostname = getenv("HOSTNAME");
+      if (!hostname) { hostname = getenv("HOST"); }
+      string host_id = regex_replace(hostname ? hostname : "(unknown)",
+                                     regex("[0-9]*$"), "");
+      cout << ',' << host_id; // 3
+      cout << ',' << (fes.GetNRanks() + 7)/8; // 4 (assuming 8 ranks/node !!)
+      cout << ',' << fes.GetNRanks(); // 5
+      cout << ',' << nx << ',' << ny << ',' << nz; // 6,7,8
+      cout << ',' << order; // 9
+      // DiffusionMultigrid::ConstructBilinearForm p+1+q1d_inc 1D points
+      real_t Q1D = order + 1 + q1d_inc;
+      cout << ',' << defaultfloat << Q1D; // 10 (note: written as real_t)
+      cout << ',' << scientific << epsy << ',' << epsz; // 11,12
+      cout << ',' << ndof; // 13
+      cout << ',' << niter; // 14
+      cout << ',' << monitor.initial_nrm << ',' << monitor.final_nrm; // 15,16
+      cout << ',' << L2_err; // 17
+      // cout << ',' << grad_err; // 17 *** for testing ***
+      cout << ',' << t_setup << ',' << t_solve; // 18,19
+      cout << endl;
+   }
+
+   return 0;
+}
+
+void report_hypre_gpu_status(bool gpu_aware_mpi_requested)
+{
+#ifdef HYPRE_WITH_GPU_AWARE_MPI
+   bool hypre_gpu_aware_mpi = true;
+#else
+   bool hypre_gpu_aware_mpi = false;
+#endif
+#if (MFEM_HYPRE_VERSION > 23000)
+   hypre_gpu_aware_mpi = hypre_gpu_aware_mpi && hypre_GetGpuAwareMPI();
+#endif
+   if (Mpi::Root())
+   {
+      MFEM_VERIFY(!gpu_aware_mpi_requested || hypre_gpu_aware_mpi,
+                  "GPU-aware MPI requested but HYPRE's GPU-aware MPI support"
+                  " is not enabled");
+      cout << "\nHYPRE GPU support: "
+#ifdef HYPRE_USING_GPU
+           << "enabled";
+#else
+           << "disabled";
+#endif
+      cout << "\nHYPRE GPU-aware MPI support: "
+           << (hypre_gpu_aware_mpi ? "enabled" : "disabled") << endl;
+   }
+}
+
+void report_env_vars()
+{
+   const int myid = Mpi::WorldRank();
+   // const int lastid = min(Mpi::WorldSize(),4)-1; // show up to 4 ranks
+   const int lastid = Mpi::WorldSize()-1;
+   if (myid > lastid) { return; }
+   Array<char> recv_buf;
+   int buflen = -1, tag = 42;
+   const char *env_vars[] =
+   {
+      "HOST", "HOSTNAME", "MPICH_GPU_SUPPORT_ENABLED", "CUDA_VISIBLE_DEVICES",
+      "ROCR_VISIBLE_DEVICES"
+   };
+   const int num_env_vars = sizeof(env_vars)/sizeof(env_vars[0]);
+   // Send strings to rank 0, so that they can be printed in order, guaranteed.
+   // Every rank > 0 sends to rank 0:
+   if (myid > 0)
+   {
+      for (int ev = 0; ev < num_env_vars; ev++)
+      {
+         const char *env_var_val = getenv(env_vars[ev]);
+         buflen = env_var_val ? int(strlen(env_var_val)+1) : -1;
+         MPI_Send(&buflen, 1, MPI_INT, 0, tag, MPI_COMM_WORLD);
+         if (env_var_val)
+         {
+            MPI_Send(env_var_val, buflen, MPI_CHAR, 0, tag, MPI_COMM_WORLD);
+         }
+      }
+   }
+   else // myid == 0
+   {
+      cout << "\nDefined environment variables:\n";
+      for (int id = 0; id <= lastid; id++)
+      {
+         cout << "[rank " << id << "]:";
+         for (int ev = 0, vars_shown = 0; ev < num_env_vars; ev++)
+         {
+            const char *env_var_val = nullptr;
+            if (id == 0)
+            {
+               env_var_val = getenv(env_vars[ev]);
+               buflen = env_var_val ? 0 : -1;
+            }
+            else
+            {
+               MPI_Recv(&buflen, 1, MPI_INT, id, tag, MPI_COMM_WORLD,
+                        MPI_STATUS_IGNORE);
+            }
+            if (buflen != -1)
+            {
+               if (id > 0)
+               {
+                  recv_buf.SetSize(buflen);
+                  MPI_Recv(recv_buf.begin(), buflen, MPI_CHAR, id, tag,
+                           MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+                  env_var_val = recv_buf.begin();
+               }
+               if (vars_shown)
+               {
+                  cout << "\n[rank " << id << "]:";
+               }
+               cout << ' ' << env_vars[ev] << '=' << env_var_val;
+               vars_shown++;
+            }
+         }
+         cout << '\n';
+      }
+      if (lastid < Mpi::WorldSize()-1)
+      {
+         cout << "... [only " << lastid+1 << '/' << Mpi::WorldSize()
+              << " ranks shown]\n";
+      }
+      cout << flush;
+   }
+}
+
+real_t verify_ess_bdr(const Vector &b, const Vector &x,
+                      const Array<int> &ess_tdof_list)
+{
+   Vector d(ess_tdof_list.Size());
+   auto d_b = b.Read();
+   auto d_x = x.Read();
+   auto d_d = d.Write();
+   auto d_ess_ind = ess_tdof_list.Read();
+   mfem::forall(ess_tdof_list.Size(), [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int ind = d_ess_ind[i];
+      d_d[i] = -fabs(d_b[ind] - d_x[ind]);
+   });
+   real_t d_max = -d.Min(); // max is not implemented on device
+   MPI_Allreduce(MPI_IN_PLACE, &d_max, 1, MFEM_MPI_REAL_T, MPI_MAX,
+                 MPI_COMM_WORLD);
+   return d_max;
+}

miniapps/meshing/fit-node-position.cpp

@@ -224,5 +224,6 @@ int main (int argc, char *argv[])
    }
 
    delete metric;
+   delete fec_mesh;
    return 0;
 }

miniapps/solvers/block-solvers.cpp

@@ -52,12 +52,13 @@
 //        (respectively 0), essential (respectively natural) boundary condition
 //        will be imposed on boundary with the i-th attribute.
 
+#include <fstream>
+#include <iostream>
+#include <functional>
+
 #include "mfem.hpp"
 #include "bramble_pasciak.hpp"
 #include "div_free_solver.hpp"
-#include <fstream>
-#include <iostream>
-#include <memory>
 
 using namespace std;
 using namespace mfem;
@@ -83,48 +84,54 @@ real_t natural_bc(const Vector & x);
        D: subset of the boundary where natural boundary condition is imposed. */
 class DarcyProblem
 {
-   OperatorPtr M_;
-   OperatorPtr B_;
-   Vector rhs_;
-   Vector ess_data_;
-   ParGridFunction u_;
-   ParGridFunction p_;
+   OperatorPtr M_, B_;
+   Vector rhs_, ess_data_;
+   ParGridFunction u_, p_;
    ParMesh mesh_;
-   ParBilinearForm *mVarf_;
-   ParMixedBilinearForm *bVarf_;
+   DFSSpaces dfs_spaces_;
+   std::function<bool (int)> refine_fn = [&](int num_refs)
+   {
+      for (int l = 0; l < num_refs; l++)
+      {
+         mesh_.UniformRefinement();
+         dfs_spaces_.CollectDFSData();
+      }
+      return true;
+   };
+   const bool dfs_refine_;
+   ParBilinearForm mVarf_;
+   ParMixedBilinearForm bVarf_;
    VectorFunctionCoefficient ucoeff_;
    FunctionCoefficient pcoeff_;
-   DFSSpaces dfs_spaces_;
    PWConstCoefficient mass_coeff;
    const IntegrationRule *irs_[Geometry::NumGeom];
 public:
    DarcyProblem(Mesh &mesh, int num_refines, int order, const char *coef_file,
                 Array<int> &ess_bdr, DFSParameters param);
 
-   HypreParMatrix& GetM() { return *M_.As<HypreParMatrix>(); }
-   HypreParMatrix& GetB() { return *B_.As<HypreParMatrix>(); }
+   const HypreParMatrix& GetM() const { return *M_.As<HypreParMatrix>(); }
+   const HypreParMatrix& GetB() const { return *B_.As<HypreParMatrix>(); }
    const Vector& GetRHS() { return rhs_; }
    const Vector& GetEssentialBC() { return ess_data_; }
    const DFSData& GetDFSData() const { return dfs_spaces_.GetDFSData(); }
    void ShowError(const Vector &sol, bool verbose);
    void VisualizeSolution(const Vector &sol, std::string tag, int visport = 19916);
-   ParBilinearForm* GetMform() const { return mVarf_; }
-   ParMixedBilinearForm* GetBform() const { return bVarf_; }
+   ParBilinearForm& GetMform() { return mVarf_; }
+   ParMixedBilinearForm& GetBform() { return bVarf_; }
 };
 
 DarcyProblem::DarcyProblem(Mesh &mesh, int num_refs, int order,
                            const char *coef_file, Array<int> &ess_bdr,
                            DFSParameters dfs_param)
-   : mesh_(MPI_COMM_WORLD, mesh), ucoeff_(mesh.Dimension(), u_exact),
-     pcoeff_(p_exact), dfs_spaces_(order, num_refs, &mesh_, ess_bdr, dfs_param),
+   : mesh_(MPI_COMM_WORLD, mesh),
+     dfs_spaces_(order, num_refs, &mesh_, ess_bdr, dfs_param),
+     dfs_refine_(refine_fn(num_refs)),
+     mVarf_(dfs_spaces_.GetHdivFES()),
+     bVarf_(dfs_spaces_.GetHdivFES(), dfs_spaces_.GetL2FES()),
+     ucoeff_(mesh.Dimension(), u_exact),
+     pcoeff_(p_exact),
      mass_coeff()
 {
-   for (int l = 0; l < num_refs; l++)
-   {
-      mesh_.UniformRefinement();
-      dfs_spaces_.CollectDFSData();
-   }
-
    Vector coef_vector(mesh.GetNE());
    coef_vector = 1.0;
    if (std::strcmp(coef_file, ""))
@@ -153,24 +160,20 @@ DarcyProblem::DarcyProblem(Mesh &mesh, int num_refs, int order,
    gform.AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
    gform.Assemble();
 
-   mVarf_ = new ParBilinearForm(dfs_spaces_.GetHdivFES());
-   bVarf_ = new ParMixedBilinearForm(dfs_spaces_.GetHdivFES(),
-                                     dfs_spaces_.GetL2FES());
+   mVarf_.AddDomainIntegrator(new VectorFEMassIntegrator(mass_coeff));
+   mVarf_.ComputeElementMatrices();
+   mVarf_.Assemble();
+   mVarf_.EliminateEssentialBC(ess_bdr, u_, fform);
 
-   mVarf_->AddDomainIntegrator(new VectorFEMassIntegrator(mass_coeff));
-   mVarf_->ComputeElementMatrices();
-   mVarf_->Assemble();
-   mVarf_->EliminateEssentialBC(ess_bdr, u_, fform);
+   mVarf_.Finalize();
+   M_.Reset(mVarf_.ParallelAssemble());
 
-   mVarf_->Finalize();
-   M_.Reset(mVarf_->ParallelAssemble());
-
-   bVarf_->AddDomainIntegrator(new VectorFEDivergenceIntegrator);
-   bVarf_->Assemble();
-   bVarf_->SpMat() *= -1.0;
-   bVarf_->EliminateTrialEssentialBC(ess_bdr, u_, gform);
-   bVarf_->Finalize();
-   B_.Reset(bVarf_->ParallelAssemble());
+   bVarf_.AddDomainIntegrator(new VectorFEDivergenceIntegrator);
+   bVarf_.Assemble();
+   bVarf_.SpMat() *= -1.0;
+   bVarf_.EliminateTrialEssentialBC(ess_bdr, u_, gform);
+   bVarf_.Finalize();
+   B_.Reset(bVarf_.ParallelAssemble());
 
    rhs_.SetSize(M_->NumRows() + B_->NumRows());
    Vector rhs_block0(rhs_.GetData(), M_->NumRows());
@@ -341,8 +344,8 @@ int main(int argc, char *argv[])
 
    // Generate components of the saddle point problem
    DarcyProblem darcy(*mesh, par_ref_levels, order, coef_file, ess_bdr, param);
-   HypreParMatrix& M = darcy.GetM();
-   HypreParMatrix& B = darcy.GetB();
+   const HypreParMatrix &M = darcy.GetM();
+   const HypreParMatrix &B = darcy.GetB();
    const DFSData& DFS_data = darcy.GetDFSData();
    delete mesh;
 
@@ -362,6 +365,7 @@ int main(int argc, char *argv[])
    std::map<const DarcySolver*, real_t> setup_time;
    chrono.Restart();
    BDPMinresSolver bdp(M, B, param);
+   bdp.iterative_mode = true;
    setup_time[&bdp] = chrono.RealTime();
 
    chrono.Restart();

miniapps/solvers/block_fespace_operator.cpp

@@ -14,29 +14,27 @@
 namespace mfem
 {
 
-BlockFESpaceOperator::BlockFESpaceOperator(const
-                                           std::vector<const FiniteElementSpace*> &fespaces):
+BlockFESpaceOperator::BlockFESpaceOperator(const FESVector &fespaces):
    Operator(GetHeight(fespaces)),
    offsets(GetBlockOffsets(fespaces)),
    prolongColOffsets(GetProColBlockOffsets(fespaces)),
    restrictRowOffsets(GetResRowBlockOffsets(fespaces)),
    A(offsets),
-   prolongation(offsets,prolongColOffsets),
+   prolongation(offsets, prolongColOffsets),
    restriction(restrictRowOffsets, offsets)
 {
    for (size_t i = 0; i <fespaces.size(); i++)
    {
       // Since const_cast is required here, be sure to avoid using
       // BlockOperator::GetBlock on restriction or prolongation.
-      prolongation.SetDiagonalBlock(i,
-                                    const_cast<Operator *>(fespaces[i]->GetProlongationMatrix()));
-      restriction.SetDiagonalBlock(i,
-                                   const_cast<Operator *>(fespaces[i]->GetRestrictionOperator()));
+      auto prolongation_matrix = fespaces[i]->GetProlongationMatrix();
+      auto restriction_matrix = fespaces[i]->GetRestrictionOperator();
+      prolongation.SetDiagonalBlock(i, const_cast<Operator *>(prolongation_matrix));
+      restriction.SetDiagonalBlock(i, const_cast<Operator *>(restriction_matrix));
    }
 }
 
-int BlockFESpaceOperator::GetHeight(const std::vector<const FiniteElementSpace*>
-                                    &fespaces)
+int BlockFESpaceOperator::GetHeight(const FESVector &fespaces)
 {
    int height = 0;
    for (size_t i = 0; i < fespaces.size(); i++)
@@ -46,8 +44,7 @@ int BlockFESpaceOperator::GetHeight(const std::vector<const FiniteElementSpace*>
    return height;
 }
 
-Array<int> BlockFESpaceOperator::GetBlockOffsets(const
-                                                 std::vector<const FiniteElementSpace*> &fespaces)
+Array<int> BlockFESpaceOperator::GetBlockOffsets(const FESVector &fespaces)
 {
    Array<int> offsets(fespaces.size()+1);
    offsets[0] = 0;
@@ -60,8 +57,8 @@ Array<int> BlockFESpaceOperator::GetBlockOffsets(const
    return offsets;
 }
 
-Array<int> BlockFESpaceOperator::GetProColBlockOffsets(const
-                                                       std::vector<const FiniteElementSpace*> &fespaces)
+Array<int> BlockFESpaceOperator::GetProColBlockOffsets(const FESVector
+                                                       &fespaces)
 {
    Array<int> offsets(fespaces.size()+1);
    offsets[0] = 0;
@@ -83,8 +80,8 @@ Array<int> BlockFESpaceOperator::GetProColBlockOffsets(const
    return offsets;
 }
 
-Array<int> BlockFESpaceOperator::GetResRowBlockOffsets(const
-                                                       std::vector<const FiniteElementSpace*> &fespaces)
+Array<int> BlockFESpaceOperator::GetResRowBlockOffsets(const FESVector
+                                                       &fespaces)
 {
    Array<int> offsets(fespaces.size()+1);
    std::cout << "fespaces.size() = " << fespaces.size() << std::endl;

miniapps/solvers/block_fespace_operator.hpp

@@ -25,7 +25,8 @@ namespace mfem
 ///        L-Vectors. For example, a block may be a BilinearForm.
 class BlockFESpaceOperator : public Operator
 {
-private:
+   using FESVector = std::vector<const FiniteElementSpace*>;
+
    /// Offsets for the square "A" operator.
    Array<int> offsets;
    /// Column offsets for the prolongation operator.
@@ -39,33 +40,27 @@ private:
    /// Maps true dofs of each block to local dofs.
    BlockOperator restriction;
    /// Computes height for parent operator.
-   static int GetHeight(const std::vector<const FiniteElementSpace*>
-                        &fespaces);
+   static int GetHeight(const FESVector &fespaces);
    /// Computes offsets for A BlockOperator.
-   static Array<int> GetBlockOffsets(const std::vector<const FiniteElementSpace*>
-                                     &fespaces);
+   static Array<int> GetBlockOffsets(const FESVector &fespaces);
    /// Computes col_offsets for prolongation operator.
-   static Array<int> GetProColBlockOffsets(const
-                                           std::vector<const FiniteElementSpace*> &fespaces);
+   static Array<int> GetProColBlockOffsets(const FESVector &fespaces);
    /// Computes row_offsets for restriction operator.
-   static Array<int> GetResRowBlockOffsets(const
-                                           std::vector<const FiniteElementSpace*> &fespaces);
+   static Array<int> GetResRowBlockOffsets(const FESVector &fespaces);
+
 public:
    /// @brief Constructor for BlockFESpaceOperator.
    /// @param[in] fespaces Finite element spaces for diagonal blocks. Spaces are not owned.
-   BlockFESpaceOperator(const std::vector<const FiniteElementSpace*> &fespaces);
+   BlockFESpaceOperator(const FESVector &fespaces);
    const Operator* GetProlongation () const override;
    const Operator* GetRestriction () const override;
    void Mult(const Vector &x, Vector &y) const override {A.Mult(x,y);};
    /// @brief Wraps BlockOperator::SetBlock. Eventually would like this class to inherit
    /// from BlockOperator instead, but can't easily due to ownership of offset data
    /// in BlockOperator being by reference.
-   void SetBlock( int   iRow,
-                  int   iCol,
-                  Operator *  op,
-                  real_t   c = 1.0) {A.SetBlock(iRow, iCol, op, c);};
+   void SetBlock(int iRow, int iCol, Operator *op, real_t c = 1.0) { A.SetBlock(iRow, iCol, op, c); };
 };
 
 } // namespace mfem
 
-#endif
+#endif // MFEM_BLOCK_FESPACE_OPERATOR

miniapps/solvers/bramble_pasciak.cpp

@@ -9,70 +9,66 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-
 #include "bramble_pasciak.hpp"
 
 using namespace std;
 
-namespace mfem
-{
-namespace blocksolvers
+namespace mfem::blocksolvers
 {
+
 /// Bramble-Pasciak Solver
-BramblePasciakSolver::BramblePasciakSolver(
-   ParBilinearForm *mVarf,
-   ParMixedBilinearForm *bVarf,
-   const BPSParameters &param)
-   : DarcySolver(mVarf->ParFESpace()->GetTrueVSize(),
-                 bVarf->TestFESpace()->GetTrueVSize())
+BramblePasciakSolver::BramblePasciakSolver(ParBilinearForm &mVarf,
+                                           ParMixedBilinearForm &bVarf,
+                                           const BPSParameters &param)
+   : DarcySolver(mVarf.ParFESpace()->GetTrueVSize(),
+                 bVarf.TestFESpace()->GetTrueVSize())
 {
-   M_.reset(mVarf->ParallelAssemble());
-   B_.reset(bVarf->ParallelAssemble());
-   Q_.reset(ConstructMassPreconditioner(*mVarf, param.q_scaling));
+   M_.reset(mVarf.ParallelAssemble());
+   B_.reset(bVarf.ParallelAssemble());
+   Q_.reset(ConstructMassPreconditioner(mVarf, param.q_scaling));
 
    Vector diagM;
    M_->GetDiag(diagM);
-   auto BT = B_->Transpose();
-   auto invDBt = new HypreParMatrix(*BT);
+   std::unique_ptr<HypreParMatrix> invDBt(B_->Transpose());
    invDBt->InvScaleRows(diagM);
-   auto S = ParMult(B_.get(), invDBt);
+   S_.reset(ParMult(B_.get(), invDBt.get(), true));
+   invDBt.reset();
    M0_.Reset(new HypreDiagScale(*M_));
-   M1_.Reset(new HypreBoomerAMG(*S));
+   M1_.Reset(new HypreBoomerAMG(*S_));
    M1_.As<HypreBoomerAMG>()->SetPrintLevel(0);
 
    Init(*M_, *B_, *Q_, *M0_.As<Solver>(), *M1_.As<Solver>(), param);
 }
 
-BramblePasciakSolver::BramblePasciakSolver(
-   HypreParMatrix &M, HypreParMatrix &B, HypreParMatrix &Q,
-   Solver &M0, Solver &M1,
-   const BPSParameters &param)
+BramblePasciakSolver::BramblePasciakSolver(HypreParMatrix &M,
+                                           HypreParMatrix &B,
+                                           HypreParMatrix &Q,
+                                           Solver &M0, Solver &M1,
+                                           const BPSParameters &param)
    : DarcySolver(M.NumRows(), B.NumRows())
 {
    Init(M, B, Q, M0, M1, param);
 }
 
-void BramblePasciakSolver::Init(
-   HypreParMatrix &M, HypreParMatrix &B, HypreParMatrix &Q,
-   Solver &M0, Solver &M1,
-   const BPSParameters &param)
+void BramblePasciakSolver::Init(HypreParMatrix &M,
+                                HypreParMatrix &B,
+                                HypreParMatrix &Q,
+                                Solver &M0, Solver &M1,
+                                const BPSParameters &param)
 {
-   auto Bt = new TransposeOperator(&B);
+   Bt_ = std::make_unique<TransposeOperator>(&B);
    auto invQ = new HypreDiagScale(Q);
 
    use_bpcg = param.use_bpcg;
 
    if (use_bpcg)
    {
-      oop_ = new BlockOperator(offsets_);
-      oop_->owns_blocks = false;
+      oop_ = std::make_unique<BlockOperator>(offsets_);
       oop_->SetBlock(0, 0, &M);
-      oop_->SetBlock(0, 1, Bt);
+      oop_->SetBlock(0, 1, Bt_.get());
       oop_->SetBlock(1, 0, &B);
-
       // cpc_ unused in bpcg
       auto temp_cpc = new BlockDiagonalPreconditioner(offsets_);
-      temp_cpc->owns_blocks = true;
       temp_cpc->SetDiagonalBlock(0, invQ);
       temp_cpc->SetDiagonalBlock(1, &M1);
       // tri(1,0) = B M0 = B invQ
@@ -81,51 +77,48 @@ void BramblePasciakSolver::Init(
       auto BinvQ = new ProductOperator(&B, invQ, false, false);
       // tri
       auto temp_tri = new BlockOperator(offsets_);
-      temp_tri->owns_blocks = true;
       temp_tri->SetBlock(0, 0, id_m);
       temp_tri->SetBlock(1, 1, id_b, -1.0);
       temp_tri->SetBlock(1, 0, BinvQ);
+      temp_tri->owns_blocks = 1;
 
-      ppc_ = new ProductOperator(temp_cpc, temp_tri, true, true);
+      ppc_ = std::make_unique<ProductOperator>(temp_cpc, temp_tri, true, true);
 
-      ipc_ = new BlockOperator(offsets_);
-      ipc_->owns_blocks = false;
+      ipc_ = std::make_unique<BlockOperator>(offsets_);
       ipc_->SetDiagonalBlock(0, invQ);
+      ipc_->owns_blocks = 1;
 
       // bpcg
-      solver_.reset(new BPCGSolver(M.GetComm(), *ipc_, *ppc_));
+      solver_ = std::make_unique<BPCGSolver>(M.GetComm(), ipc_.get(), ppc_.get());
       solver_->SetOperator(*oop_);
    }
    else
    {
       // oop_ unused in cg
       auto temp_oop = new BlockOperator(offsets_);
-      temp_oop->owns_blocks = false;
       temp_oop->SetBlock(0, 0, &M);
-      temp_oop->SetBlock(0, 1, Bt);
+      temp_oop->SetBlock(0, 1, Bt_.get());
       temp_oop->SetBlock(1, 0, &B);
 
       // ipc_ unused in cg
       auto temp_ipc = new BlockOperator(offsets_);
-      temp_ipc->owns_blocks = false;
       temp_ipc->SetDiagonalBlock(0, invQ);
+      temp_ipc->owns_blocks = 1;
 
       // temp_AN = temp_oop * temp_ipc
       auto temp_AN = new ProductOperator(temp_oop, temp_ipc, true, true);
 
       // Required for updating the RHS
       auto id = new IdentityOperator(M.NumRows()+B.NumRows());
-      map_ = new SumOperator(temp_AN, 1.0, id, -1.0, true, true);
+      map_ = std::make_unique<SumOperator>(temp_AN, 1.0, id, -1.0, true, true);
+      mop_ = std::make_unique<ProductOperator>(map_.get(), temp_oop, false, false);
 
-      mop_ = new ProductOperator(map_, temp_oop, false, true);
-
-      cpc_ = new BlockDiagonalPreconditioner(offsets_);
-      cpc_->owns_blocks = true;
+      cpc_ = std::make_unique<BlockDiagonalPreconditioner>(offsets_);
       cpc_->SetDiagonalBlock(0, &M0);
       cpc_->SetDiagonalBlock(1, &M1);
 
       // (P)CG
-      solver_.reset(new CGSolver(M.GetComm()));
+      solver_ = std::make_unique<CGSolver>(M.GetComm());
       solver_->SetOperator(*mop_);
       solver_->SetPreconditioner(*cpc_);
    }
@@ -133,7 +126,7 @@ void BramblePasciakSolver::Init(
 }
 
 HypreParMatrix *BramblePasciakSolver::ConstructMassPreconditioner(
-   ParBilinearForm &mVarf, real_t q_scaling)
+   const ParBilinearForm &mVarf, real_t q_scaling)
 {
    MFEM_ASSERT((q_scaling > 0.0) && (q_scaling < 1.0),
                "Invalid Q-scaling factor: q_scaling = " << q_scaling );
@@ -167,7 +160,7 @@ HypreParMatrix *BramblePasciakSolver::ConstructMassPreconditioner(
       Vector x(M_i.Height()), Mx(M_i.Height()), diff(M_i.Height());
       real_t eval_prev = 0.0;
       int iter = 0;
-      x.Randomize(696383552+779345*i);
+      x.Randomize(static_cast<int>(696383552LL+779345LL*i));
 #if defined(MFEM_USE_DOUBLE)
       const real_t rel_tol = 1e-12;
 #elif defined(MFEM_USE_SINGLE)
@@ -400,5 +393,5 @@ void BPCGSolver::Mult(const Vector &b, Vector &x) const
    final_norm = sqrt(delta);
    Monitor(final_iter, final_norm, r, x, true);
 }
-} // namespace blocksolvers
-} // namespace mfem
+
+} // namespace mfem::blocksolvers

miniapps/solvers/bramble_pasciak.hpp

@@ -49,9 +49,7 @@
 #include "darcy_solver.hpp"
 #include <memory>
 
-namespace mfem
-{
-namespace blocksolvers
+namespace mfem::blocksolvers
 {
 
 /// Parameters for the BramblePasciakSolver method
@@ -70,11 +68,11 @@ protected:
    void UpdateVectors();
 
 public:
-   BPCGSolver(const Operator &ipc, const Operator &ppc) { pprec = &ppc; iprec = &ipc; }
+   BPCGSolver(const Operator *ipc, const Operator *ppc): iprec(ipc), pprec(ppc) {}
 
 #ifdef MFEM_USE_MPI
-   BPCGSolver(MPI_Comm comm_, const Operator &ipc, const Operator &ppc)
-      : IterativeSolver(comm_) { pprec = &ppc; iprec = &ipc; }
+   BPCGSolver(MPI_Comm comm_, const Operator *ipc, const Operator *ppc)
+      : IterativeSolver(comm_), iprec(ipc), pprec(ppc) { }
 #endif
 
    void SetOperator(const Operator &op) override
@@ -83,11 +81,9 @@ public:
    void SetPreconditioner(Solver &pc) override
    { if (Mpi::Root()) { MFEM_WARNING("SetPreconditioner has no effect on BPCGSolver.\n"); } }
 
-   virtual void SetIncompletePreconditioner(const Operator &ipc)
-   { iprec = &ipc; }
+   virtual void SetIncompletePreconditioner(const Operator *ipc) { iprec = ipc; }
 
-   virtual void SetParticularPreconditioner(const Operator &ppc)
-   { pprec = &ppc; }
+   virtual void SetParticularPreconditioner(const Operator *ppc) { pprec = ppc; }
 
    void Mult(const Vector &b, Vector &x) const override;
 };
@@ -116,23 +112,20 @@ public:
     1. P. Vassilevski, Multilevel Block Factorization Preconditioners (Appendix
        F.3), Springer, 2008.
 
-    2. J. Bramble and J. Pasciak.  A Preconditioning Technique for Indefinite
+    2. J. Bramble and J. Pasciak. A Preconditioning Technique for Indefinite
        Systems Resulting From Mixed Approximations of Elliptic Problems,
        Mathematics of Computation, 50:1-17, 1988. */
 class BramblePasciakSolver : public DarcySolver
 {
    mutable bool use_bpcg;
    std::unique_ptr<IterativeSolver> solver_;
-   BlockOperator *oop_, *ipc_;
-   ProductOperator *mop_;
-   SumOperator *map_;
-   ProductOperator *ppc_;
-   BlockDiagonalPreconditioner *cpc_;
-   std::unique_ptr<HypreParMatrix> M_;
-   std::unique_ptr<HypreParMatrix> B_;
-   std::unique_ptr<HypreParMatrix> Q_;
-   OperatorPtr M0_;
-   OperatorPtr M1_;
+   std::unique_ptr<BlockOperator> oop_, ipc_;
+   std::unique_ptr<ProductOperator> mop_, ppc_;
+   std::unique_ptr<SumOperator> map_;
+   std::unique_ptr<BlockDiagonalPreconditioner> cpc_;
+   std::unique_ptr<HypreParMatrix> M_, B_, Q_, S_;
+   std::unique_ptr<TransposeOperator> Bt_;
+   OperatorPtr M0_, M1_;
    Array<int> ess_zero_dofs_;
 
    void Init(HypreParMatrix &M, HypreParMatrix &B,
@@ -142,8 +135,8 @@ class BramblePasciakSolver : public DarcySolver
 public:
    /// System and mass preconditioner are constructed from bilinear forms
    BramblePasciakSolver(
-      ParBilinearForm *mVarf,
-      ParMixedBilinearForm *bVarf,
+      ParBilinearForm &mVarf,
+      ParMixedBilinearForm &bVarf,
       const BPSParameters &param);
 
    /// System and mass preconditioner are user-provided
@@ -158,8 +151,8 @@ public:
        element T:
                                 M_T x_T = lambda_T diag(M_T) x_T.
        We set Q_T = alpha * min(lambda_T) * diag(M_T), 0 < alpha < 1. */
-   static HypreParMatrix *ConstructMassPreconditioner(ParBilinearForm &mVarf,
-                                                      real_t alpha = 0.5);
+   static HypreParMatrix *ConstructMassPreconditioner(const ParBilinearForm &mVarf,
+                                                      const real_t alpha = 0.5);
 
    void Mult(const Vector &x, Vector &y) const override;
    void SetOperator(const Operator &op) override { }
@@ -167,7 +160,6 @@ public:
    int GetNumIterations() const override { return solver_->GetNumIterations(); }
 };
 
-} // namespace blocksolvers
-} // namespace mfem
+} // namespace mfem::blocksolvers
 
 #endif // MFEM_BP_SOLVER_HPP

miniapps/solvers/darcy_solver.cpp

@@ -13,10 +13,9 @@
 
 using namespace std;
 
-namespace mfem
-{
-namespace blocksolvers
+namespace mfem::blocksolvers
 {
+
 void SetOptions(IterativeSolver& solver, const IterSolveParameters& param)
 {
    solver.SetPrintLevel(param.print_level);
@@ -49,7 +48,7 @@ BDPMinresSolver::BDPMinresSolver(const HypreParMatrix& M,
    prec_.SetDiagonalBlock(0, new HypreDiagScale(M));
    prec_.SetDiagonalBlock(1, new HypreBoomerAMG(*S_.As<HypreParMatrix>()));
    static_cast<HypreBoomerAMG&>(prec_.GetDiagonalBlock(1)).SetPrintLevel(0);
-   prec_.owns_blocks = true;
+   prec_.owns_blocks = 1;
 
    SetOptions(solver_, param);
    solver_.SetOperator(op_);
@@ -58,8 +57,9 @@ BDPMinresSolver::BDPMinresSolver(const HypreParMatrix& M,
 
 void BDPMinresSolver::Mult(const Vector & x, Vector & y) const
 {
+   solver_.iterative_mode = this->iterative_mode;
    solver_.Mult(x, y);
    for (int dof : ess_zero_dofs_) { y[dof] = 0.0; }
 }
-} // namespace blocksolvers
-} // namespace mfem
+
+} // namespace mfem::blocksolvers

miniapps/solvers/darcy_solver.hpp

@@ -13,13 +13,10 @@
 #define MFEM_DARCY_SOLVER_HPP
 
 #include "mfem.hpp"
-#include <memory>
-#include <vector>
 
-namespace mfem
-{
-namespace blocksolvers
+namespace mfem::blocksolvers
 {
+
 struct IterSolveParameters
 {
    int print_level = 0;
@@ -32,8 +29,6 @@ struct IterSolveParameters
    real_t rel_tol = 1e-5;
 #else
 #error "Only single and double precision are supported!"
-   real_t abs_tol = 1e-12;
-   real_t rel_tol = 1e-9;
 #endif
 };
 
@@ -57,7 +52,7 @@ class BDPMinresSolver : public DarcySolver
    BlockDiagonalPreconditioner prec_;
    OperatorPtr BT_;
    OperatorPtr S_;   // S_ = B diag(M)^{-1} B^T
-   MINRESSolver solver_;
+   mutable MINRESSolver solver_;
    Array<int> ess_zero_dofs_;
 public:
    BDPMinresSolver(const HypreParMatrix& M,
@@ -68,7 +63,7 @@ public:
    void SetEssZeroDofs(const Array<int>& dofs) { dofs.Copy(ess_zero_dofs_); }
    int GetNumIterations() const override { return solver_.GetNumIterations(); }
 };
-} // namespace blocksolvers
-} // namespace mfem
+
+} // namespace mfem::blocksolvers
 
 #endif // MFEM_DARCY_SOLVER_HPP

miniapps/solvers/div_free_solver.cpp

@@ -13,16 +13,16 @@
 
 using namespace std;
 
-namespace mfem
+namespace mfem::blocksolvers
 {
-namespace blocksolvers
+
+static HypreParMatrix* TwoStepsRAP(const HypreParMatrix *Rt,
+                                   const HypreParMatrix *A,
+                                   const HypreParMatrix *P)
 {
-HypreParMatrix* TwoStepsRAP(const HypreParMatrix& Rt, const HypreParMatrix& A,
-                            const HypreParMatrix& P)
-{
-   OperatorPtr R(Rt.Transpose());
-   OperatorPtr RA(ParMult(R.As<HypreParMatrix>(), &A));
-   return ParMult(RA.As<HypreParMatrix>(), &P, true);
+   OperatorPtr R(Rt->Transpose());
+   OperatorPtr RA(ParMult(R.As<HypreParMatrix>(), A));
+   return ParMult(RA.As<HypreParMatrix>(), P, true);
 }
 
 void GetRowColumnsRef(const SparseMatrix& A, int row, Array<int>& cols)
@@ -59,34 +59,35 @@ DFSSpaces::DFSSpaces(int order, int num_refine, ParMesh *mesh,
 
    if (mesh->Dimension() == 3)
    {
-      hcurl_fec_.reset(new ND_FECollection(order+1, mesh->Dimension()));
+      hcurl_fec_ = std::make_unique<ND_FECollection>(order+1, mesh->Dimension());
    }
    else
    {
-      hcurl_fec_.reset(new H1_FECollection(order+1, mesh->Dimension()));
+      hcurl_fec_ = std::make_unique<H1_FECollection>(order+1, mesh->Dimension());
    }
 
    all_bdr_attr_.SetSize(ess_attr.Size(), 1);
-   hdiv_fes_.reset(new ParFiniteElementSpace(mesh, &hdiv_fec_));
-   l2_fes_.reset(new ParFiniteElementSpace(mesh, &l2_fec_));
-   coarse_hdiv_fes_.reset(new ParFiniteElementSpace(*hdiv_fes_));
-   coarse_l2_fes_.reset(new ParFiniteElementSpace(*l2_fes_));
-   l2_0_fes_.reset(new ParFiniteElementSpace(mesh, &l2_0_fec_));
+   hdiv_fes_ = std::make_unique<ParFiniteElementSpace>(mesh, &hdiv_fec_);
+   l2_fes_ = std::make_unique<ParFiniteElementSpace>(mesh, &l2_fec_);
+   coarse_hdiv_fes_ = std::make_unique<ParFiniteElementSpace>(*hdiv_fes_);
+   coarse_l2_fes_ = std::make_unique<ParFiniteElementSpace>(*l2_fes_);
+   l2_0_fes_ = std::make_unique<ParFiniteElementSpace>(mesh, &l2_0_fec_);
    l2_0_fes_->SetUpdateOperatorType(Operator::MFEM_SPARSEMAT);
    el_l2dof_.reserve(num_refine+1);
    el_l2dof_.push_back(ElemToDof(*coarse_l2_fes_));
 
    data_.agg_hdivdof.resize(num_refine);
    data_.agg_l2dof.resize(num_refine);
-   data_.P_hdiv.resize(num_refine, OperatorPtr(Operator::Hypre_ParCSR));
-   data_.P_l2.resize(num_refine, OperatorPtr(Operator::Hypre_ParCSR));
+   data_.P_hdiv.resize(num_refine);
+   data_.P_l2.resize(num_refine);
+
    data_.Q_l2.resize(num_refine);
    hdiv_fes_->GetEssentialTrueDofs(ess_attr, data_.coarsest_ess_hdivdofs);
    data_.C.resize(num_refine+1);
 
-   hcurl_fes_.reset(new ParFiniteElementSpace(mesh, hcurl_fec_.get()));
-   coarse_hcurl_fes_.reset(new ParFiniteElementSpace(*hcurl_fes_));
-   data_.P_hcurl.resize(num_refine, OperatorPtr(Operator::Hypre_ParCSR));
+   hcurl_fes_ = std::make_unique<ParFiniteElementSpace>(mesh, hcurl_fec_.get());
+   coarse_hcurl_fes_ = std::make_unique<ParFiniteElementSpace>(*hcurl_fes_);
+   data_.P_hcurl.resize(num_refine);
 }
 
 SparseMatrix* AggToInteriorDof(const Array<int>& bdr_truedofs,
@@ -104,8 +105,8 @@ SparseMatrix* AggToInteriorDof(const Array<int>& bdr_truedofs,
    agg_tdof_T.As<HypreParMatrix>()->GetDiag(tdof_agg);
    agg_tdof_T.As<HypreParMatrix>()->GetOffd(is_shared, trash);
 
-   int * I = new int [tdof_agg.NumRows()+1]();
-   int * J = new int[tdof_agg.NumNonZeroElems()];
+   int *I = new int[tdof_agg.NumRows()+1]();
+   int *J = new int[tdof_agg.NumNonZeroElems()];
 
    Array<int> is_bdr;
    FiniteElementSpace::ListToMarker(bdr_truedofs, tdof_agg.NumRows(), is_bdr);
@@ -119,7 +120,7 @@ SparseMatrix* AggToInteriorDof(const Array<int>& bdr_truedofs,
       J[counter++] = tdof_agg.GetRowColumns(i)[0];
    }
 
-   real_t * D = new real_t[I[tdof_agg.NumRows()]];
+   auto *D = new real_t[I[tdof_agg.NumRows()]];
    std::fill_n(D, I[tdof_agg.NumRows()], 1.0);
 
    SparseMatrix intdof_agg(I, J, D, tdof_agg.NumRows(), tdof_agg.NumCols());
@@ -146,20 +147,21 @@ void DFSSpaces::MakeDofRelationTables(int level)
 
 void DFSSpaces::CollectDFSData()
 {
-   auto GetP = [this](OperatorPtr& P, unique_ptr<ParFiniteElementSpace>& cfes,
-                      ParFiniteElementSpace& fes, bool remove_zero)
+   auto GetP = [&](std::unique_ptr<OperatorPtr> &P,
+                   std::unique_ptr<ParFiniteElementSpace> &cfes,
+                   ParFiniteElementSpace& fes, const bool remove_zero)
    {
       fes.Update();
-      fes.GetTrueTransferOperator(*cfes, P);
-      if (remove_zero)
-      {
-         P.As<HypreParMatrix>()->DropSmallEntries(1e-16);
-      }
+      auto T = new OperatorHandle(Operator::Hypre_ParCSR);
+      fes.GetTrueTransferOperator(*cfes, *T);
+      P.reset(T);
+      if (remove_zero) { P->As<HypreParMatrix>()->DropSmallEntries(1e-16); }
       (level_ < (int)data_.P_l2.size()-1) ? cfes->Update() : cfes.reset();
    };
 
    GetP(data_.P_hdiv[level_], coarse_hdiv_fes_, *hdiv_fes_, true);
    GetP(data_.P_l2[level_], coarse_l2_fes_, *l2_fes_, false);
+
    MakeDofRelationTables(level_);
 
    GetP(data_.P_hcurl[level_], coarse_hcurl_fes_, *hcurl_fes_, true);
@@ -171,7 +173,9 @@ void DFSSpaces::CollectDFSData()
    data_.C[level_+1].Reset(curl.ParallelAssemble());
    mfem::Array<int> ess_hcurl_tdof;
    hcurl_fes_->GetEssentialTrueDofs(ess_bdr_attr_, ess_hcurl_tdof);
-   data_.C[level_+1].As<HypreParMatrix>()->EliminateCols(ess_hcurl_tdof);
+   HypreParMatrix *res =
+      data_.C[level_+1].As<HypreParMatrix>()->EliminateCols(ess_hcurl_tdof);
+   delete res;
 
    ++level_;
 
@@ -189,7 +193,7 @@ void DFSSpaces::DataFinalize()
    SparseMatrix P_l2;
    for (int l = (int)data_.P_l2.size()-1; l >= 0; --l)
    {
-      data_.P_l2[l].As<HypreParMatrix>()->GetDiag(P_l2);
+      data_.P_l2[l]->As<HypreParMatrix>()->GetDiag(P_l2);
       OperatorPtr PT_l2(Transpose(P_l2));
       auto PTW = Mult(*PT_l2.As<SparseMatrix>(), *W.As<SparseMatrix>());
       auto cW = Mult(*PTW, P_l2);
@@ -245,7 +249,7 @@ SaddleSchwarzSmoother::SaddleSchwarzSmoother(const HypreParMatrix& M,
                                              const SparseMatrix& agg_hdivdof,
                                              const SparseMatrix& agg_l2dof,
                                              const HypreParMatrix& P_l2,
-                                             const HypreParMatrix& Q_l2)
+                                             const ProductOperator& Q_l2)
    : Solver(M.NumRows() + B.NumRows()), agg_hdivdof_(agg_hdivdof),
      agg_l2dof_(agg_l2dof), solvers_loc_(agg_l2dof.NumRows())
 {
@@ -312,23 +316,27 @@ void SaddleSchwarzSmoother::Mult(const Vector & x, Vector & y) const
    blk_y.GetBlock(1) -= coarse_l2_projection;
 }
 
-DivFreeSolver::DivFreeSolver(const HypreParMatrix &M, const HypreParMatrix& B,
+DivFreeSolver::DivFreeSolver(const HypreParMatrix &M,
+                             const HypreParMatrix &B,
                              const DFSData& data)
    : DarcySolver(M.NumRows(), B.NumRows()), data_(data), param_(data.param),
-     BT_(B.Transpose()), BBT_solver_(B, param_.BBT_solve_param),
-     ops_offsets_(data.P_l2.size()+1), ops_(ops_offsets_.size()),
-     blk_Ps_(ops_.Size()-1), smoothers_(ops_.Size())
+     BT_(B.Transpose()),
+     BBT_solver_(B, param_.BBT_solve_param),
+     ops_offsets_(data.P_l2.size()+1),
+     ops_(ops_offsets_.size()),
+     blk_Ps_(ops_.size()-1),
+     smoothers_(ops_.size())
 {
    ops_offsets_.back().MakeRef(DarcySolver::offsets_);
-   ops_.Last() = new BlockOperator(ops_offsets_.back());
-   ops_.Last()->SetBlock(0, 0, const_cast<HypreParMatrix*>(&M));
-   ops_.Last()->SetBlock(1, 0, const_cast<HypreParMatrix*>(&B));
-   ops_.Last()->SetBlock(0, 1, BT_.Ptr());
+   ops_.back() = std::make_unique<BlockOperator>(ops_offsets_.back());
+   ops_.back()->SetBlock(0, 0, const_cast<HypreParMatrix*>(&M));
+   ops_.back()->SetBlock(1, 0, const_cast<HypreParMatrix*>(&B));
+   ops_.back()->SetBlock(0, 1, BT_.Ptr());
 
    for (int l = data.P_l2.size(); l >= 0; --l)
    {
-      auto& M_f = static_cast<const HypreParMatrix&>(ops_[l]->GetBlock(0, 0));
-      auto& B_f = static_cast<const HypreParMatrix&>(ops_[l]->GetBlock(1, 0));
+      auto &M_f = static_cast<const HypreParMatrix&>(ops_[l]->GetBlock(0, 0));
+      auto &B_f = static_cast<const HypreParMatrix&>(ops_[l]->GetBlock(1, 0));
 
       if (l == 0)
       {
@@ -343,123 +351,112 @@ DivFreeSolver::DivFreeSolver(const HypreParMatrix &M, const HypreParMatrix& B,
 
          const IterSolveParameters& param = param_.coarse_solve_param;
          auto coarse_solver = new BDPMinresSolver(M_f, B_f, param);
-         if (ops_.Size() > 1)
+         if (ops_.size() > 1)
          {
             coarse_solver->SetEssZeroDofs(data.coarsest_ess_hdivdofs);
          }
-         smoothers_[l] = coarse_solver;
+         smoothers_[l].reset(coarse_solver);
          continue;
       }
 
-      HypreParMatrix& P_hdiv_l = *data.P_hdiv[l-1].As<HypreParMatrix>();
-      HypreParMatrix& P_l2_l = *data.P_l2[l-1].As<HypreParMatrix>();
+      auto P_hdiv_l = data.P_hdiv[l-1]->As<HypreParMatrix>();
+      auto P_l2_l = data.P_l2[l-1]->As<HypreParMatrix>();
       SparseMatrix& agg_hdivdof_l = *data.agg_hdivdof[l-1].As<SparseMatrix>();
       SparseMatrix& agg_l2dof_l = *data.agg_l2dof[l-1].As<SparseMatrix>();
-      HypreParMatrix& Q_l2_l = *data.Q_l2[l-1].As<HypreParMatrix>();
-      HypreParMatrix* C_l = data.C[l].As<HypreParMatrix>();
+      ProductOperator& Q_l2_l = *data.Q_l2[l-1].As<ProductOperator>();
+      auto* C_l = data.C[l].As<HypreParMatrix>();
 
       auto S0 = new SaddleSchwarzSmoother(M_f, B_f, agg_hdivdof_l,
-                                          agg_l2dof_l, P_l2_l, Q_l2_l);
+                                          agg_l2dof_l, *P_l2_l, Q_l2_l);
       if (param_.coupled_solve)
       {
          auto S1 = new BlockDiagonalPreconditioner(ops_offsets_[l]);
          S1->SetDiagonalBlock(0, new AuxSpaceSmoother(M_f, C_l));
-         S1->owns_blocks = true;
-         smoothers_[l] = new ProductSolver(ops_[l], S0, S1, false, true, true);
+         S1->owns_blocks = 1;
+         smoothers_[l] =
+            std::make_unique<ProductSolver>(ops_[l].get(), S0, S1, false, true, true);
       }
       else
       {
-         smoothers_[l] = S0;
+         smoothers_[l].reset(S0);
       }
 
-      HypreParMatrix* M_c = TwoStepsRAP(P_hdiv_l, M_f, P_hdiv_l);
-      HypreParMatrix* B_c = TwoStepsRAP(P_l2_l, B_f, P_hdiv_l);
+      HypreParMatrix* M_c = TwoStepsRAP(P_hdiv_l, &M_f, P_hdiv_l);
+      HypreParMatrix* B_c = TwoStepsRAP(P_l2_l, &B_f, P_hdiv_l);
 
       ops_offsets_[l-1].SetSize(3, 0);
       ops_offsets_[l-1][1] = M_c->NumRows();
       ops_offsets_[l-1][2] = M_c->NumRows() + B_c->NumRows();
 
-      blk_Ps_[l-1] = new BlockOperator(ops_offsets_[l], ops_offsets_[l-1]);
-      blk_Ps_[l-1]->SetBlock(0, 0, &P_hdiv_l);
-      blk_Ps_[l-1]->SetBlock(1, 1, &P_l2_l);
+      blk_Ps_[l-1] =
+         std::make_unique<BlockOperator>(ops_offsets_[l], ops_offsets_[l-1]);
+      blk_Ps_[l-1]->SetBlock(0, 0, P_hdiv_l);
+      blk_Ps_[l-1]->SetBlock(1, 1, P_l2_l);
 
-      ops_[l-1] = new BlockOperator(ops_offsets_[l-1]);
+      ops_[l-1] =
+         std::make_unique<BlockOperator>(ops_offsets_[l-1]);
       ops_[l-1]->SetBlock(0, 0, M_c);
       ops_[l-1]->SetBlock(1, 0, B_c);
       ops_[l-1]->SetBlock(0, 1, B_c->Transpose());
-      ops_[l-1]->owns_blocks = true;
+      ops_[l-1]->owns_blocks = 1;
    }
 
-   Array<bool> own_ops(ops_.Size());
-   Array<bool> own_smoothers(smoothers_.Size());
-   Array<bool> own_Ps(blk_Ps_.Size());
-   own_ops = true;
-   own_smoothers = true;
-   own_Ps = true;
-
    if (data_.P_l2.size() == 0) { return; }
 
+   Array<bool> own_ops(ops_.size());
+   Array<bool> own_smoothers(smoothers_.size());
+   Array<bool> own_blk_Ps(blk_Ps_.size());
+   own_ops = false, own_smoothers = false, own_blk_Ps = false;
+
+   Array<Solver*> smoothers(smoothers_.size());
+
    if (param_.coupled_solve)
    {
       solver_.Reset(new GMRESSolver(B.GetComm()));
-      solver_.As<GMRESSolver>()->SetOperator(*(ops_.Last()));
-      prec_.Reset(new Multigrid(ops_, smoothers_, blk_Ps_,
-                                own_ops, own_smoothers, own_Ps));
+      solver_.As<GMRESSolver>()->SetOperator(*(ops_.back()));
+      Array<BlockOperator*> ops(ops_.size()), blk_Ps(blk_Ps_.size());
+      for (size_t i = 0; i < ops_.size(); ++i) { ops[i] = ops_[i].get(); }
+      for (size_t i = 0; i < blk_Ps_.size(); ++i) { blk_Ps[i] = blk_Ps_[i].get(); }
+      for (size_t i = 0; i < smoothers_.size(); ++i) { smoothers[i] = smoothers_[i].get(); }
+      prec_.Reset(new Multigrid(ops, smoothers, blk_Ps,
+                                own_ops, own_smoothers, own_blk_Ps));
    }
    else
    {
       Array<HypreParMatrix*> ops(data_.P_hcurl.size()+1);
-      Array<Solver*> smoothers(ops.Size());
       Array<HypreParMatrix*> Ps(data_.P_hcurl.size());
-      own_Ps = false;
-
-      HypreParMatrix& C_finest = *data.C.back().As<HypreParMatrix>();
-      ops.Last() = TwoStepsRAP(C_finest, M, C_finest);
+      auto C_finest = data.C.back().As<HypreParMatrix>();
+      ops.Last() = TwoStepsRAP(C_finest, &M, C_finest);
       ops.Last()->EliminateZeroRows();
       ops.Last()->DropSmallEntries(1e-14);
-
       solver_.Reset(new CGSolver(B.GetComm()));
       solver_.As<CGSolver>()->SetOperator(*ops.Last());
       smoothers.Last() = new HypreSmoother(*ops.Last());
       static_cast<HypreSmoother*>(smoothers.Last())->SetOperatorSymmetry(true);
-
       for (int l = Ps.Size()-1; l >= 0; --l)
       {
-         Ps[l] = data_.P_hcurl[l].As<HypreParMatrix>();
-         ops[l] = TwoStepsRAP(*Ps[l], *ops[l+1], *Ps[l]);
+         Ps[l] = data_.P_hcurl[l]->As<HypreParMatrix>();
+         ops[l] = TwoStepsRAP(Ps[l], ops[l+1], Ps[l]);
          ops[l]->DropSmallEntries(1e-14);
          smoothers[l] = new HypreSmoother(*ops[l]);
          static_cast<HypreSmoother*>(smoothers[l])->SetOperatorSymmetry(true);
       }
-
-      prec_.Reset(new Multigrid(ops, smoothers, Ps, own_ops, own_smoothers, own_Ps));
+      own_ops = true, own_smoothers = true;
+      prec_.Reset(new Multigrid(ops, smoothers, Ps,
+                                own_ops, own_smoothers, own_blk_Ps));
    }
 
    solver_.As<IterativeSolver>()->SetPreconditioner(*prec_.As<Solver>());
    SetOptions(*solver_.As<IterativeSolver>(), param_);
 }
 
-DivFreeSolver::~DivFreeSolver()
-{
-   if (param_.coupled_solve) { return; }
-   for (int i = 0; i < ops_.Size(); ++i)
-   {
-      delete ops_[i];
-      delete smoothers_[i];
-      if (i == ops_.Size() - 1) { break; }
-      delete blk_Ps_[i];
-   }
-}
-
 void DivFreeSolver::SolveParticular(const Vector& rhs, Vector& sol) const
 {
-   std::vector<Vector> rhss(smoothers_.Size());
-   std::vector<Vector> sols(smoothers_.Size());
-
+   std::vector<Vector> rhss(smoothers_.size()), sols(smoothers_.size());
    rhss.back().SetDataAndSize(const_cast<real_t*>(rhs.HostRead()), rhs.Size());
    sols.back().SetDataAndSize(sol.HostWrite(), sol.Size());
 
-   for (int l = blk_Ps_.Size()-1; l >= 0; --l)
+   for (int l = blk_Ps_.size()-1; l >= 0; --l)
    {
       rhss[l].SetSize(blk_Ps_[l]->NumCols());
       sols[l].SetSize(blk_Ps_[l]->NumCols());
@@ -470,12 +467,12 @@ void DivFreeSolver::SolveParticular(const Vector& rhs, Vector& sol) const
       blk_Ps_[l]->MultTranspose(rhss[l+1], rhss[l]);
    }
 
-   for (int l = 0; l < smoothers_.Size(); ++l)
+   for (size_t l = 0; l < smoothers_.size(); ++l)
    {
       smoothers_[l]->Mult(rhss[l], sols[l]);
    }
 
-   for (int l = 0; l < blk_Ps_.Size(); ++l)
+   for (size_t l = 0; l < blk_Ps_.size(); ++l)
    {
       Vector P_sol(blk_Ps_[l]->NumRows());
       blk_Ps_[l]->Mult(sols[l], P_sol);
@@ -507,12 +504,12 @@ void DivFreeSolver::Mult(const Vector & x, Vector & y) const
    MFEM_VERIFY(x.Size() == offsets_[2], "MLDivFreeSolver: x size is invalid");
    MFEM_VERIFY(y.Size() == offsets_[2], "MLDivFreeSolver: y size is invalid");
 
-   if (ops_.Size() == 1) { smoothers_[0]->Mult(x, y); return; }
+   if (ops_.size() == 1) { smoothers_[0]->Mult(x, y); return; }
 
    BlockVector blk_y(y, offsets_);
 
    BlockVector resid(offsets_);
-   ops_.Last()->Mult(y, resid);
+   ops_.back()->Mult(y, resid);
    add(1.0, x, -1.0, resid, resid);
 
    BlockVector correction(offsets_);
@@ -539,7 +536,7 @@ void DivFreeSolver::Mult(const Vector & x, Vector & y) const
       ch.Clear();
       ch.Start();
 
-      ops_.Last()->Mult(y, resid);
+      ops_.back()->Mult(y, resid);
       add(1.0, x, -1.0, resid, resid);
 
       SolveDivFree(resid.GetBlock(0), correction.GetBlock(0));
@@ -553,7 +550,7 @@ void DivFreeSolver::Mult(const Vector & x, Vector & y) const
       ch.Clear();
       ch.Start();
 
-      auto& M = dynamic_cast<const HypreParMatrix&>(ops_.Last()->GetBlock(0, 0));
+      auto& M = dynamic_cast<const HypreParMatrix&>(ops_.back()->GetBlock(0, 0));
       M.Mult(-1.0, correction.GetBlock(0), 1.0, resid.GetBlock(0));
       SolvePotential(resid.GetBlock(0), correction.GetBlock(1));
       blk_y.GetBlock(1) += correction.GetBlock(1);
@@ -567,11 +564,12 @@ void DivFreeSolver::Mult(const Vector & x, Vector & y) const
 
 int DivFreeSolver::GetNumIterations() const
 {
-   if (ops_.Size() == 1)
+   if (ops_.size() == 1)
    {
-      return static_cast<BDPMinresSolver*>(smoothers_[0])->GetNumIterations();
+      return static_cast<BDPMinresSolver*>
+             (smoothers_.at(0).get())->GetNumIterations();
    }
    return solver_.As<IterativeSolver>()->GetNumIterations();
 }
-} // namespace blocksolvers
-} // namespace mfem
+
+} // namespace mfem::blocksolvers

miniapps/solvers/div_free_solver.hpp

@@ -13,11 +13,11 @@
 #define MFEM_DIVFREE_SOLVER_HPP
 
 #include "darcy_solver.hpp"
+#include <memory>
 
-namespace mfem
-{
-namespace blocksolvers
+namespace mfem::blocksolvers
 {
+
 /// Parameters for the divergence free solver
 struct DFSParameters : IterSolveParameters
 {
@@ -35,14 +35,16 @@ struct DFSParameters : IterSolveParameters
 /// Data for the divergence free solver
 struct DFSData
 {
-   std::vector<OperatorPtr> agg_hdivdof;  // agglomerates to H(div) dofs table
-   std::vector<OperatorPtr> agg_l2dof;    // agglomerates to L2 dofs table
-   std::vector<OperatorPtr> P_hdiv;   // Interpolation matrix for H(div) space
-   std::vector<OperatorPtr> P_l2;     // Interpolation matrix for L2 space
-   std::vector<OperatorPtr> P_hcurl;  // Interpolation for kernel space of div
-   std::vector<OperatorPtr> Q_l2;     // Q_l2[l] = (W_{l+1})^{-1} P_l2[l]^T W_l
-   Array<int> coarsest_ess_hdivdofs;  // coarsest level essential H(div) dofs
-   std::vector<OperatorPtr> C;        // discrete curl: ND -> RT, map to Null(B)
+   using UniqueOperatorPtr = std::unique_ptr<OperatorPtr>;
+
+   std::vector<OperatorPtr> agg_hdivdof; // agglomerates to H(div) dofs table
+   std::vector<OperatorPtr> agg_l2dof; // agglomerates to L2 dofs table
+   std::vector<UniqueOperatorPtr> P_hdiv; // Interpolation matrix for H(div) space
+   std::vector<UniqueOperatorPtr> P_l2; // Interpolation matrix for L2 space
+   std::vector<UniqueOperatorPtr> P_hcurl; // Interpolation for kernel space of div
+   std::vector<OperatorPtr> Q_l2; // Q_l2[l] = (W_{l+1})^{-1} P_l2[l]^T W_l
+   Array<int> coarsest_ess_hdivdofs; // coarsest level essential H(div) dofs
+   std::vector<OperatorPtr> C; // discrete curl: ND -> RT, map to Null(B)
    DFSParameters param;
 };
 
@@ -92,8 +94,7 @@ public:
 /// Compute the product B * B^T and solve it with CG preconditioned by BoomerAMG
 class BBTSolver : public Solver
 {
-   OperatorPtr BBT_;
-   OperatorPtr BBT_prec_;
+   OperatorPtr BBT_, BBT_prec_;
    CGSolver BBT_solver_;
 public:
    BBTSolver(const HypreParMatrix &B, IterSolveParameters param);
@@ -115,14 +116,11 @@ public:
 ///                      [ B   0  ]
 class SaddleSchwarzSmoother : public Solver
 {
-   const SparseMatrix& agg_hdivdof_;
-   const SparseMatrix& agg_l2dof_;
+   const SparseMatrix &agg_hdivdof_, &agg_l2dof_;
    OperatorPtr coarse_l2_projector_;
 
    Array<int> offsets_;
-   mutable Array<int> offsets_loc_;
-   mutable Array<int> hdivdofs_loc_;
-   mutable Array<int> l2dofs_loc_;
+   mutable Array<int> offsets_loc_, hdivdofs_loc_, l2dofs_loc_;
    std::vector<OperatorPtr> solvers_loc_;
 public:
    /** SaddleSchwarzSmoother solves local saddle point problems defined on a
@@ -140,7 +138,7 @@ public:
                          const SparseMatrix& agg_hdivdof,
                          const SparseMatrix& agg_l2dof,
                          const HypreParMatrix& P_l2,
-                         const HypreParMatrix& Q_l2);
+                         const ProductOperator& Q_l2);
    void Mult(const Vector &x, Vector &y) const override;
    void MultTranspose(const Vector &x, Vector &y) const override { Mult(x, y); }
    void SetOperator(const Operator &op) override { }
@@ -178,11 +176,10 @@ class DivFreeSolver : public DarcySolver
    OperatorPtr BT_;
    BBTSolver BBT_solver_;
    std::vector<Array<int>> ops_offsets_;
-   Array<BlockOperator*> ops_;
-   Array<BlockOperator*> blk_Ps_;
-   Array<Solver*> smoothers_;
-   OperatorPtr prec_;
-   OperatorPtr solver_;
+   std::vector<std::unique_ptr<BlockOperator>> ops_;
+   std::vector<std::unique_ptr<BlockOperator>> blk_Ps_;
+   std::vector<std::unique_ptr<Solver>> smoothers_;
+   OperatorPtr prec_, solver_;
 
    void SolveParticular(const Vector& rhs, Vector& sol) const;
    void SolveDivFree(const Vector& rhs, Vector& sol) const;
@@ -190,14 +187,11 @@ class DivFreeSolver : public DarcySolver
 public:
    DivFreeSolver(const HypreParMatrix& M, const HypreParMatrix &B,
                  const DFSData& data);
-   ~DivFreeSolver();
    void Mult(const Vector &x, Vector &y) const override;
    void SetOperator(const Operator &op) override { }
    int GetNumIterations() const override;
 };
 
-} // namespace blocksolvers
-
-} // namespace mfem
+} // namespace mfem::blocksolvers
 
 #endif // MFEM_DIVFREE_SOLVER_HPP

miniapps/solvers/plor_solvers.cpp

@@ -1,69 +1,3 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-//
-//                 ------------------------------------------
-//                 Parallel Low-Order Refined Solvers Miniapp
-//                 ------------------------------------------
-//
-// This miniapp illustrates the use of low-order refined preconditioners for
-// finite element problems defined using H1, H(curl), H(div), or L2 finite
-// element spaces. The following problems are solved, depending on the chosen
-// finite element space:
-//
-// H1 and L2: definite Helmholtz problem, u - Delta u = f
-//            (in L2 discretized using the symmetric interior penalty DG method)
-//
-// H(curl):   definite Maxwell problem, u + curl curl u = f
-//
-// H(div):    grad-div problem, u - grad(div u) = f
-//
-// In each case, the high-order finite element problem is preconditioned using a
-// low-order finite element discretization defined on a Gauss-Lobatto refined
-// mesh. The low-order problem is solved using hypre's AMG preconditioners:
-// BoomerAMG is used for H1 and L2 problems, AMS is used for H(curl) and 2D
-// H(div) problems, and ADS is used for 3D H(div) problems.
-//
-// For vector finite element spaces, the special "Integrated" basis type is used
-// to obtain spectral equivalence between the high-order and low-order refined
-// discretizations. This basis is defined in reference [1] and spectral
-// equivalence is shown in [2]:
-//
-// [1]. M. Gerritsma. Edge functions for spectral element methods. Spectral and
-//      High Order Methods for Partial Differential Equations. (2010)
-// [2]. C. Dohrmann. Spectral equivalence properties of higher-order tensor
-//      product finite elements and applications to preconditioning. (2021)
-//
-// The action of the high-order operator is computed using MFEM's partial
-// assembly/matrix-free algorithms (except in the case of L2, which remains
-// future work).
-//
-// Compile with: make plor_solvers
-//
-// Sample runs:
-//
-//    mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe h
-//    mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe n
-//    mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe r
-//    mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe l
-//    mpirun -np 4 plor_solvers -m ../../data/amr-hex.mesh -fe h -rs 0 -o 2
-//    mpirun -np 4 plor_solvers -m ../../data/star-surf.mesh -fe h
-//    mpirun -np 4 plor_solvers -m ../../data/star-surf.mesh -fe n
-//    mpirun -np 4 plor_solvers -m ../../data/star-surf.mesh -fe r
-//
-// Device sample runs:
-//  * mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe h -d cuda
-//  * mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe n -d cuda
-//  * mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe r -d cuda
-//  * mpirun -np 4 plor_solvers -m ../../data/fichera.mesh -fe l -d cuda
-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -74,54 +8,35 @@
 using namespace std;
 using namespace mfem;
 
-int main(int argc, char *argv[])
+struct Opts
 {
-   Mpi::Init();
-   Hypre::Init();
-
    const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = 1, par_ref_levels = 1;
+   int ser_ref_levels = 1;
+   int par_ref_levels = 1;
    int order = 3;
    const char *fe = "h";
-   const char *device_config = "cpu";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order", "Polynomial degree.");
-   args.AddOption(&fe, "-fe", "--fe-type",
-                  "FE type. h for H1, n for Hcurl, r for Hdiv, l for L2");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.ParseCheck();
-
-   Device device(device_config);
-   if (Mpi::Root()) { device.Print(); }
+};
 
+int Run(const Opts &opts)
+{
    bool H1 = false, ND = false, RT = false, L2 = false;
-   if (string(fe) == "h") { H1 = true; }
-   else if (string(fe) == "n") { ND = true; }
-   else if (string(fe) == "r") { RT = true; }
-   else if (string(fe) == "l") { L2 = true; }
+   if (string(opts.fe) == "h") { H1 = true; }
+   else if (string(opts.fe) == "n") { ND = true; }
+   else if (string(opts.fe) == "r") { RT = true; }
+   else if (string(opts.fe) == "l") { L2 = true; }
    else { MFEM_ABORT("Bad FE type. Must be 'h', 'n', 'r', or 'l'."); }
 
-   real_t kappa = (order+1)*(order+1); // Penalty used for DG discretizations
+   const int order = opts.order;
+   const real_t kappa = (order+1)*(order+1); // Penalty used for DG discretizations
 
-   Mesh serial_mesh(mesh_file, 1, 1);
+   Mesh serial_mesh(opts.mesh_file, 1, 1);
    const int dim = serial_mesh.Dimension();
    const int sdim = serial_mesh.SpaceDimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "Mesh dimension must be 2 or 3.");
    MFEM_VERIFY(!L2 || dim == sdim, "DG surface meshes not supported.");
-   for (int l = 0; l < ser_ref_levels; l++) { serial_mesh.UniformRefinement(); }
+   for (int l = 0; l < opts.ser_ref_levels; l++) { serial_mesh.UniformRefinement(); }
    ParMesh mesh(MPI_COMM_WORLD, serial_mesh);
-   for (int l = 0; l < par_ref_levels; l++) { mesh.UniformRefinement(); }
+   for (int l = 0; l < opts.par_ref_levels; l++) { mesh.UniformRefinement(); }
    serial_mesh.Clear();
 
    if (mesh.ncmesh && (RT || ND))
@@ -139,6 +54,18 @@ int main(int argc, char *argv[])
    else { fec.reset(new L2_FECollection(order, dim, b1)); }
 
    ParFiniteElementSpace fes(&mesh, fec.get());
+
+   // fes.GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
+
+   {
+      MFEM_PERF_SCOPE("Ensure Nodes");
+      mesh.EnsureNodes();
+   }
+   // {
+   //    auto &ir = DiffusionIntegrator::GetRule(*fes.GetFE(0), *fes.GetFE(0));
+   //    mesh.GetGeometricFactors(ir, GeometricFactors::JACOBIANS);
+   // }
+
    HYPRE_Int ndofs = fes.GlobalTrueVSize();
    if (Mpi::Root()) { cout << "Number of DOFs: " << ndofs << endl; }
 
@@ -149,7 +76,7 @@ int main(int argc, char *argv[])
    ParBilinearForm a(&fes);
    if (H1 || L2)
    {
-      a.AddDomainIntegrator(new MassIntegrator);
+      // a.AddDomainIntegrator(new MassIntegrator);
       a.AddDomainIntegrator(new DiffusionIntegrator);
    }
    else
@@ -167,7 +94,9 @@ int main(int argc, char *argv[])
    // Partial assembly not currently supported for DG or for surface meshes with
    // vector finite elements (ND or RT).
    if (!L2 && (H1 || sdim == dim)) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+
    a.Assemble();
+   // a.Assemble();
 
    ParLinearForm b(&fes);
    if (H1 || L2) { b.AddDomainIntegrator(new DomainLFIntegrator(f_coeff)); }
@@ -188,9 +117,13 @@ int main(int argc, char *argv[])
    a.FormLinearSystem(ess_dofs, x, b, A, X, B);
 
    unique_ptr<Solver> solv_lor;
+
    if (H1 || L2)
    {
-      solv_lor.reset(new LORSolver<HypreBoomerAMG>(a, ess_dofs));
+      auto solv = new LORSolver<HypreBoomerAMG>(a, ess_dofs);
+      solv->GetSolver().SetPrintLevel(0);
+      solv->GetSolver().Setup(B, X);
+      solv_lor.reset(solv);
    }
    else if (RT && dim == 3)
    {
@@ -212,30 +145,53 @@ int main(int argc, char *argv[])
 
    a.RecoverFEMSolution(X, b, x);
 
-   if (sdim == dim)
-   {
-      real_t er =
-         (H1 || L2) ? x.ComputeL2Error(u_coeff) : x.ComputeL2Error(u_vec_coeff);
-      if (Mpi::Root()) { cout << "L2 error: " << er << endl; }
-   }
-
-   if (visualization)
-   {
-      // Save the solution and mesh to disk. The output can be viewed using
-      // GLVis as follows: "glvis -np <np> -m mesh -g sol"
-      x.Save("sol");
-      mesh.Save("mesh");
-
-      // Also save the solution for visualization using ParaView
-      ParaViewDataCollection dc("PLOR", &mesh);
-      dc.SetPrefixPath("ParaView");
-      dc.SetHighOrderOutput(true);
-      dc.SetLevelsOfDetail(order);
-      dc.RegisterField("u", &x);
-      dc.SetCycle(0);
-      dc.SetTime(0.0);
-      dc.Save();
-   }
-
    return 0;
 }
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   Opts opts;
+   const char *device_config = "cpu";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&opts.mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&opts.ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&opts.par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&opts.order, "-o", "--order", "Polynomial degree.");
+   args.AddOption(&opts.fe, "-fe", "--fe-type",
+                  "FE type. h for H1, n for Hcurl, r for Hdiv, l for L2");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.ParseCheck();
+
+   Device device(device_config);
+   if (Mpi::Root()) { device.Print(); }
+
+   MFEM_PERF_SYNC(true);
+
+   {
+      MFEM_PERF_SCOPE("Temporary allocations");
+      Vector tmp1(1024 * 1024 * 1024);
+      tmp1.ReadWrite();
+      Vector tmp2(1024 * 1024 * 1024);
+      tmp2.ReadWrite();
+   }
+   {
+      MFEM_PERF_SCOPE("Hypre allocations");
+      double *tmp1 = mfem_hypre_CTAlloc(double, 1024 * 1024 * 1024);
+      double *tmp2 = mfem_hypre_CTAlloc(double, 1024 * 1024 * 1024);
+      mfem_hypre_TFree(tmp2);
+      mfem_hypre_TFree(tmp1);
+   }
+
+   MFEM_PERF_DISABLE;
+   Run(opts);
+
+   MFEM_PERF_ENABLE;
+   Run(opts);
+}

miniapps/spde/generate_random_field.cpp

@@ -33,8 +33,6 @@
 //     (2D random field with anisotropy)
 //     mpirun -np 4 generate_random_field -o 1 -r 3 -rp 3 -nu 4 -l1 0.09 -l2 0.03 -l3 0.05 -s 0.01 -t 0.08 -top 1 -no-rs -m ../../data/ref-square.mesh
 
-#include <math.h>
-#include <fstream>
 #include <iostream>
 #include <string>
 #include "mfem.hpp"
@@ -260,7 +258,8 @@ int main(int argc, char *argv[])
    // III.3 Solve the SPDE problem
    spde::SPDESolver solver(nu, bc, &fespace, l1, l2, l3, e1, e2,
                            e3);
-   const int seed = (random_seed) ? 0 : std::numeric_limits<int>::max();
+   const int seed = (random_seed) ? 0 :
+                    std::numeric_limits<int>::max() - Mpi::WorldRank();
    solver.SetupRandomFieldGenerator(seed);
    solver.GenerateRandomField(u);
 

miniapps/toys/lissajous.cpp

@@ -133,8 +133,11 @@ int main(int argc, char *argv[])
       u.Save(sol_ofs);
    }
 
-   soutv << "keys '.0" << std::string((int)b, '0') << "'\n" << flush;
-   south << "keys '.0" << std::string((int)a, '0') << "'\n" << flush;
+   if (visualization)
+   {
+      soutv << "keys '.0" << std::string((int)b, '0') << "'\n" << flush;
+      south << "keys '.0" << std::string((int)a, '0') << "'\n" << flush;
+   }
 
    cout << "Which direction(s) are the two curves spinning in?\n";
 

tests/benchmarks/CMakeLists.txt

@@ -31,11 +31,6 @@ function(add_benchmark name)
        set_property(SOURCE ${${NAME}_BENCH_SRCS} PROPERTY LANGUAGE CUDA)
     endif(MFEM_USE_CUDA)
 
-    if (MFEM_USE_HIP)
-        set_property(SOURCE ${${NAME}_BENCH_SRCS} PROPERTY LANGUAGE 
-                     HIP_SOURCE_PROPERTY_FORMAT TRUE)
-    endif(MFEM_USE_HIP)
-
     add_executable(bench_${name} ${${NAME}_BENCH_SRCS})
     target_link_libraries(bench_${name} mfem pthread)
     add_dependencies(${MFEM_ALL_BENCHMARKS_TARGET_NAME} bench_${name})
@@ -61,3 +56,8 @@ add_benchmark(elasticity)
 add_benchmark(tmop)
 add_benchmark(vector)
 add_benchmark(virtuals)
+
+if (MFEM_USE_MPI)
+    mfem_add_executable(pbench_ceed pbench_ceed.cpp)
+    target_link_libraries(pbench_ceed mfem pthread)
+endif()

tests/benchmarks/bench.hpp

@@ -57,6 +57,14 @@ public:
    }
 };
 
+// Reporter with no output, used with MPI on non-root ranks.
+struct NoReporter : public benchmark::BenchmarkReporter
+{
+   explicit NoReporter() = default;
+   bool ReportContext(const Context &) override { return true; }
+   void ReportRuns(const std::vector<Run> &) override {}
+};
+
 } // namespace mfem
 
 #endif // MFEM_USE_BENCHMARK

tests/benchmarks/makefile

@@ -22,7 +22,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_TESTS = bench_assembly_levels bench_ceed bench_dg_amr bench_elasticity \
             bench_tmop bench_vector bench_virtuals
-PAR_TESTS = 
+PAR_TESTS = pbench_ceed
 ifeq ($(MFEM_USE_MPI),NO)
    TESTS = $(SEQ_TESTS)
 else
@@ -43,7 +43,7 @@ all: $(TESTS)
 # Rules for building the TESTS
 
 %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+	$(MFEM_CXX) $(MFEM_FLAGS) -I$(MFEM_DIR) $< -o $@ $(MFEM_LIBS)
 
 # Rules for compiling miniapp dependencies
 $($(TESTS)): \

tests/benchmarks/pbench_ceed.cpp

@@ -0,0 +1,606 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Compile with: make pbench_ceed
+//
+// Sample runs:
+//    mpirun -np 4 pbench_ceed
+//    mpirun -np 4 pbench_ceed --benchmark_filter=BP3
+//    mpirun -np 4 pbench_ceed --benchmark_filter=BP3 --benchmark_context=local_size=5e4
+//    mpirun -np 6 pbench_ceed --benchmark_filter=BP3 --benchmark_context=proc_grid=3x2x1,local_size=5e4
+//
+// Device sample runs:
+//    mpirun -np 4 pbench_ceed --benchmark_context=device=cuda,local_size=1e6
+//    mpirun -np 4 pbench_ceed --benchmark_filter=BP3 --benchmark_context=device=cuda,local_size=1e7
+//
+// Description:
+//    This benchmark contains the implementation of the CEED's bake-off
+//    problems, BP1-BP6, and bake-off kernels, BK1-BK6: high-order benchmarks
+//    designed to test and compare the performance of high-order codes.
+//
+//    See: ceed.exascaleproject.org/bps and github.com/CEED/benchmarks
+
+
+#include "bench.hpp"
+
+#ifdef MFEM_USE_BENCHMARK
+
+// for instantiating more kernels:
+#include <fem/integ/bilininteg_mass_kernels.hpp>
+#include <fem/integ/bilininteg_diffusion_kernels.hpp>
+
+// Global parameters
+
+// local_size: desired approximate MPI-local problem size; this local size and
+// the polynomial order determine the local mesh size so that the resulting
+// problem size is (approximately) equal to local_size for all polynomial
+// orders, see MakeParMesh().
+double local_size = 61*61*61; // exact size match for all p=1,...,6
+
+// proc_grid: use processor grid given by proc_grid[0,1,2].
+int proc_grid[3] = {0,0,0};
+
+// q_gl_inc: increment for the number of GL points: q = p + 1 + q_gl_inc
+int q_gl_inc = 0;
+
+// q_gll_inc: increment for the number of GLL points: q = p + 1 + q_gll_inc
+int q_gll_inc = 0;
+
+// verbose: verbosity level: 0, 1, 2
+int verbose = 0;
+
+
+// If running on GPU, wait for GPU tasks to finish:
+inline void DeviceSync()
+{
+   if (Device::Allows(Backend::DEVICE_MASK & ~Backend::DEBUG_DEVICE))
+   {
+      MFEM_STREAM_SYNC;
+      // MFEM_DEVICE_SYNC;
+   }
+}
+
+void MakeExp2ProcGrid(int np)
+{
+   proc_grid[0] = proc_grid[1] = proc_grid[2] = 1;
+   for ( ; np >= 8; np /= 8)
+   {
+      proc_grid[0] *= 2; proc_grid[1] *= 2; proc_grid[2] *= 2;
+   }
+   if (np == 4) { proc_grid[0] *= 2; proc_grid[1] *= 2; }
+   else if (np == 2) { proc_grid[0] *= 2; }
+}
+
+// Construct the parallel mesh based on the polynomial order, p, and the
+// local_size:
+ParMesh MakeParMesh(int p)
+{
+   int nx = 0, ny = 0, nz = 0;
+   int par_ref = 0;
+   if (verbose && Mpi::Root()) { std::cout << _MFEM_FUNC_NAME << std::endl; }
+
+   const double s = local_size;
+
+   int m = floor((pow(s, 1./3)-1)/p);
+   double s_l, s_u, s_c;
+   while ((s_l=(  m  *p+1)*(  m  *p+1)*(  m  *p+1), s_l > s))  { m--; }
+   m = std::max(m, 1);
+   while ((s_u=((m+1)*p+1)*((m+1)*p+1)*((m+1)*p+1), s_u <= s)) { m++; }
+   s_l = (m*p+1)*(m*p+1)*(m*p+1);
+   if ((s_c=((m+1)*p+1)*(m*p+1)*(m*p+1), s_c > s))
+   {
+      if (s/s_l <= s_c/s) { nx = m; ny = m; nz = m; }
+      else                { nx = m; ny = m; nz = m + 1; }
+   }
+   else if ((s_l=s_c, s_c=((m+1)*p+1)*((m+1)*p+1)*(m*p+1), s_c > s))
+   {
+      if (s/s_l <= s_c/s) { nx = m; ny = m;     nz = m + 1; }
+      else                { nx = m; ny = m + 1; nz = m + 1; }
+   }
+   else
+   {
+      s_l=s_c, s_c=s_u;
+      if (s/s_l <= s_c/s) { nx = m;     ny = m + 1; nz = m + 1; }
+      else                { nx = m + 1; ny = m + 1; nz = m + 1; }
+   }
+   while (nx%2 == 0 && ny%2 == 0 && nz%2 == 0)
+   {
+      par_ref++;
+      nx /= 2; ny /= 2; nz /= 2;
+   }
+
+   nx *= proc_grid[0];
+   ny *= proc_grid[1];
+   nz *= proc_grid[2];
+
+   if (verbose && Mpi::Root())
+   {
+      std::cout
+            << '\n'
+            << "   order: " << p << '\n'
+            << "   nx: " << nx << ", ny: " << ny << ", nz: " << nz << '\n'
+            << "   px: " << proc_grid[0] << ", py: " << proc_grid[1]
+            << ", pz: " << proc_grid[2] << '\n'
+            << "   par_ref: " << par_ref << '\n'
+            << std::endl;
+   }
+
+   StopWatch timer;
+   timer.Start();
+   double t_start = timer.RealTime();
+
+   Mesh smesh(Mesh::MakeCartesian3D(nx, ny, nz, Element::HEXAHEDRON));
+   if (verbose && Mpi::Root())
+   {
+      double t_elapsed = timer.RealTime() - t_start;
+      std::cout << "   Mesh: " << 1e3*t_elapsed << " ms" << std::endl;
+   }
+   t_start = timer.RealTime();
+   Array<int> partitioning;
+   partitioning.MakeRef(smesh.CartesianPartitioning(proc_grid), smesh.GetNE(),
+                        MemoryType::HOST, true);
+   ParMesh pmesh(MPI_COMM_WORLD, smesh, partitioning.HostRead());
+   smesh.Clear();
+   for (int i = 0; i < par_ref; i++)
+   {
+      pmesh.UniformRefinement();
+   }
+   if (verbose && Mpi::Root())
+   {
+      double t_elapsed = timer.RealTime() - t_start;
+      std::cout << "   ParMesh: " << 1e3*t_elapsed << " ms" << std::endl;
+   }
+   return pmesh;
+}
+
+template <int VDIM, bool GLL>
+struct BakeOff
+{
+   static constexpr int DIM = 3;
+   const int p, q, q_order;
+   ParMesh mesh;
+   H1_FECollection fec;
+   ParFiniteElementSpace fes;
+   const Geometry::Type geom_type;
+   IntegrationRules irs;
+   const IntegrationRule *ir;
+   ConstantCoefficient one;
+   Vector uvec;
+   VectorConstantCoefficient unit_vec;
+   const HYPRE_BigInt dofs;
+   double mdofs{};
+   StopWatch timer;
+
+   BakeOff(int p):
+      p(p),
+      q(GLL ? p + 1 + q_gll_inc : p + 1 + q_gl_inc),
+      q_order(2 * q + (GLL ? -3 : -1)),
+      mesh(MakeParMesh(p)),
+      fec(p, DIM, BasisType::GaussLobatto),
+      fes(&mesh, &fec, VDIM, VDIM == 3 ? Ordering::byVDIM : Ordering::byNODES),
+      geom_type(mesh.GetTypicalElementGeometry()),
+      irs(0, GLL ? Quadrature1D::GaussLobatto : Quadrature1D::GaussLegendre),
+      ir(&irs.Get(geom_type, q_order)),
+      one(1.0),
+      uvec(DIM),
+      unit_vec((uvec = 1.0, uvec /= uvec.Norml2(), uvec)),
+      dofs(fes.GlobalTrueVSize())
+   {
+      if (verbose && Mpi::Root())
+      {
+         std::cout << "q: " << q << ", dofs: " << dofs << std::endl;
+         // std::cout << _MFEM_FUNC_NAME << std::endl;
+      }
+      timer.Start();
+   }
+
+   virtual void benchmark(benchmark::State &state) = 0;
+
+   double SumMdofs() const { return mdofs; }
+
+   double MDofs() const { return 1e-6 * dofs; }
+};
+
+/// Bake-off Problems (BPs)
+template <typename BFI, int VDIM, bool GLL>
+struct Problem : public BakeOff<VDIM, GLL>
+{
+   const double rtol = 1e-16;
+   const int max_it = 20;
+   const int print_lvl = -1;
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   ParGridFunction x;
+   ParBilinearForm a;
+   LinearForm b;
+   OperatorPtr A;
+   Vector B, X;
+   CGSolver cg;
+   int bench_call_counter = 0;
+
+   using base = BakeOff<VDIM, GLL>;
+   using base::ir;
+   using base::one;
+   using base::mesh;
+   using base::fes;
+   using base::mdofs;
+   using base::timer;
+
+   Problem(int order):
+      BakeOff<VDIM, GLL>(order),
+      ess_bdr(mesh.bdr_attributes.Max()),
+      x(&fes),
+      a(&fes),
+      b(&fes),
+      cg(fes.GetComm())
+   {
+      if (verbose && Mpi::Root()) { std::cout << _MFEM_FUNC_NAME << std::endl; }
+      ess_bdr = 1;
+      x = 0.0;
+      fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      LinearFormIntegrator *integ;
+      if (VDIM == 1)
+      {
+         integ = new DomainLFIntegrator(this->one);
+      }
+      else
+      {
+         integ = new VectorDomainLFIntegrator(this->unit_vec);
+      }
+      integ->SetIntRule(ir);
+      b.AddDomainIntegrator(integ); // b takes ownership of integ
+      b.UseFastAssembly(true);
+      b.Assemble();
+
+      double t_start = timer.RealTime();
+      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a.AddDomainIntegrator(new BFI(one, ir));
+      a.Assemble();
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+      if (verbose && Mpi::Root())
+      {
+         double t_elapsed = timer.RealTime() - t_start;
+         std::cout << "   assemble a: " << 1e3*t_elapsed << " ms" << std::endl;
+      }
+
+      cg.SetRelTol(rtol);
+      cg.SetOperator(*A);
+      cg.SetMaxIter(max_it);
+      cg.SetPrintLevel(print_lvl);
+      cg.iterative_mode = false;
+      // warmup cg
+      cg.SetMaxIter(2);
+      cg.Mult(B, X);
+      cg.SetMaxIter(max_it);
+   }
+
+   virtual ~Problem()
+   {
+      if (verbose && Mpi::Root())
+      {
+         std::cout << _MFEM_FUNC_NAME << '\n'
+                   << "   call counter: " << bench_call_counter << '\n'
+                   << "   MDofs: " << mdofs << std::endl;
+      }
+   }
+
+   void benchmark(benchmark::State &state) override
+   {
+      if (verbose > 1 && Mpi::Root())
+      {
+         std::cout << _MFEM_FUNC_NAME << std::endl;
+      }
+
+      DeviceSync();
+      MPI_Barrier(cg.GetComm());
+      double t_start = timer.RealTime();
+
+      cg.Mult(B, X);
+
+      DeviceSync();
+      MPI_Barrier(cg.GetComm());
+      double t_elapsed = timer.RealTime() - t_start;
+      // Ensure every rank gets the same time, otherwise google-benchmark may
+      // behave differently on different ranks.
+      MPI_Bcast(&t_elapsed, 1, MPI_DOUBLE, 0, cg.GetComm());
+
+      state.SetIterationTime(t_elapsed);
+      if (verbose > 1 && Mpi::Root())
+      {
+         std::cout << "   bench time: " << 1e3*t_elapsed << " ms" << std::endl;
+      }
+
+      mdofs += this->MDofs() * cg.GetNumIterations();
+      bench_call_counter++;
+   }
+};
+
+/// Bake-off Problems (BPs)
+#define BakeOff_Problem(i, Kernel, VDIM, GLL)                        \
+   static void BP##i(bm::State &state)                               \
+   {                                                                 \
+      Problem<Kernel##Integrator, VDIM, GLL> ker(state.range(0));    \
+      for (auto z : state) { ker.benchmark(state); }                 \
+      state.counters["Num Dofs"] = ker.dofs;                         \
+      state.counters["|      Dof/s"] =                               \
+         bm::Counter(1e6*ker.SumMdofs(), bm::Counter::kIsRate);      \
+      state.counters["|   Dof/s/NP"] =                               \
+         bm::Counter(1e6*ker.SumMdofs()/ker.fes.GetNRanks(),         \
+                     bm::Counter::kIsRate);                          \
+   }                                                                 \
+   BENCHMARK(BP##i)->DenseRange(1, 6)->Unit(bm::kMillisecond)->UseManualTime();
+
+// state.counters[" Q1D"] = ker.q;
+
+
+/// BP1: scalar PCG with mass matrix, GL
+BakeOff_Problem(1, Mass, 1, false)
+
+/// BP2: vector PCG with mass matrix, GL
+BakeOff_Problem(2, VectorMass, 3, false)
+
+/// BP3: scalar PCG with stiffness matrix, GL
+BakeOff_Problem(3, Diffusion, 1, false)
+
+/// BP4: vector PCG with stiffness matrix, GL
+BakeOff_Problem(4, VectorDiffusion, 3, false)
+
+/// BP5: scalar PCG with stiffness matrix, GLL
+BakeOff_Problem(5, Diffusion, 1, true)
+
+/// BP6: vector PCG with stiffness matrix, GLL
+BakeOff_Problem(6, VectorDiffusion, 3, true)
+
+/// Bake-off Kernels (BKs)
+template <typename BFI, int VDIM, bool GLL>
+struct Kernel : public BakeOff<VDIM, GLL>
+{
+   BFI bfi;
+   Vector x, y; // input and output E-vectors
+
+   using base = BakeOff<VDIM, GLL>;
+   using base::ir;
+   using base::one;
+   using base::fes;
+   using base::mdofs;
+   using base::timer;
+
+   Kernel(int order)
+      : base(order),
+        bfi(one, ir)
+   {
+      bfi.AssemblePA(fes);
+      const Table &el2dof = fes.GetElementToDofTable();
+      const int e_size = el2dof.Size_of_connections()*fes.GetVDim();
+      x.SetSize(e_size); x.UseDevice(true);
+      y.SetSize(e_size); y.UseDevice(true);
+      x.Randomize(1);
+      x.Read();
+      // warmup
+      for (int i = 0; i < 2; i++)
+      {
+         y = 0.0;
+         bfi.AddMultPA(x, y);
+      }
+   }
+
+   void benchmark(benchmark::State &state) override
+   {
+      y = 0.0;
+
+      DeviceSync();
+      MPI_Barrier(fes.GetComm());
+      double t_start = timer.RealTime();
+
+      bfi.AddMultPA(x, y);
+
+      DeviceSync();
+      MPI_Barrier(fes.GetComm());
+      double t_elapsed = timer.RealTime() - t_start;
+      // Ensure every rank gets the same time, otherwise google-benchmark may
+      // behave differently on different ranks.
+      MPI_Bcast(&t_elapsed, 1, MPI_DOUBLE, 0, fes.GetComm());
+
+      state.SetIterationTime(t_elapsed);
+      mdofs += this->MDofs();
+   }
+};
+
+/// Generic CEED BKi
+#define BakeOff_Kernel(i, KER, VDIM, GLL)                       \
+   static void BK##i(bm::State &state)                          \
+   {                                                            \
+      Kernel<KER##Integrator, VDIM, GLL> ker(state.range(0));   \
+      for (auto z : state) { ker.benchmark(state); }            \
+      state.counters["Num Dofs"] = ker.dofs;                    \
+      state.counters["|      Dof/s"] =                          \
+         bm::Counter(1e6*ker.SumMdofs(), bm::Counter::kIsRate); \
+      state.counters["|   Dof/s/NP"] =                          \
+         bm::Counter(1e6*ker.SumMdofs()/ker.fes.GetNRanks(),    \
+                     bm::Counter::kIsRate);                     \
+   }                                                            \
+   BENCHMARK(BK##i)->DenseRange(1, 6)->Unit(bm::kMillisecond)->UseManualTime();
+
+// state.counters[" Q1D"] = ker.q;
+
+
+/// BK1: scalar E-vector-to-E-vector evaluation of mass matrix, GL
+BakeOff_Kernel(1, Mass, 1, false)
+
+/// BK2: vector E-vector-to-E-vector evaluation of mass matrix, GL
+BakeOff_Kernel(2, VectorMass, 3, false)
+
+/// BK3: scalar E-vector-to-E-vector evaluation of stiffness matrix, GL
+BakeOff_Kernel(3, Diffusion, 1, false)
+
+/// BK4: vector E-vector-to-E-vector evaluation of stiffness matrix, GL
+BakeOff_Kernel(4, VectorDiffusion, 3, false)
+
+/// BK5: scalar E-vector-to-E-vector evaluation of stiffness matrix, GLL
+BakeOff_Kernel(5, Diffusion, 1, true)
+
+/// BK6: vector E-vector-to-E-vector evaluation of stiffness matrix, GLL
+BakeOff_Kernel(6, VectorDiffusion, 3, true)
+
+
+int main(int argc, char *argv[])
+{
+   // MassIntegrator specializations by <DIM, D1D, Q1D>
+   MassIntegrator::AddSpecialization<3, 3, 3>();
+   MassIntegrator::AddSpecialization<3, 4, 4>();
+   MassIntegrator::AddSpecialization<3, 5, 5>();
+   MassIntegrator::AddSpecialization<3, 6, 6>();
+   MassIntegrator::AddSpecialization<3, 7, 7>();
+
+   // DiffusionIntegrator specializations by <DIM, D1D, Q1D>
+   DiffusionIntegrator::AddSpecialization<3, 3, 3>();
+   DiffusionIntegrator::AddSpecialization<3, 4, 4>();
+   DiffusionIntegrator::AddSpecialization<3, 5, 5>();
+   DiffusionIntegrator::AddSpecialization<3, 6, 6>();
+   DiffusionIntegrator::AddSpecialization<3, 7, 7>();
+
+   Mpi::Init();
+   Hypre::Init();
+
+   bm::Initialize(&argc, argv);
+
+   // Device setup, cpu by default
+   std::string device_config = "cpu";
+   bool gpu_aware_mpi = false;
+   std::string proc_grid_str = "";
+
+   auto global_context = bmi::GetGlobalContext();
+   if (global_context != nullptr)
+   {
+      const auto device = global_context->find("device");
+      if (device != global_context->end())
+      {
+         device_config = device->second;
+      }
+      const auto ctx_gpu_aware_mpi = global_context->find("gpu_aware_mpi");
+      if (ctx_gpu_aware_mpi != global_context->end())
+      {
+         gpu_aware_mpi = std::atoi(ctx_gpu_aware_mpi->second.c_str());
+      }
+      const auto ctx_local_size = global_context->find("local_size");
+      if (ctx_local_size != global_context->end())
+      {
+         std::size_t pos;
+         local_size = std::stof(ctx_local_size->second, &pos);
+         if (ctx_local_size->second.size() != pos)
+         {
+            if (Mpi::Root())
+            {
+               std::cout << "\nerror reading local_size: "
+                         << ctx_local_size->second << '\n' << std::endl;
+            }
+            return 1;
+         }
+         if (local_size < 64.0 || local_size > std::exp2(30.0))
+         {
+            if (Mpi::Root())
+            {
+               std::cout << "\nlocal_size must be in [2^6,2^30]! local_size: "
+                         << local_size << '\n' << std::endl;
+            }
+            return 1;
+         }
+      }
+      const auto ctx_proc_grid = global_context->find("proc_grid");
+      if (ctx_proc_grid != global_context->end())
+      {
+         proc_grid_str = ctx_proc_grid->second;
+      }
+      const auto ctx_verbose = global_context->find("verbose");
+      if (ctx_verbose != global_context->end())
+      {
+         verbose = std::atoi(ctx_verbose->second.c_str());
+      }
+   }
+   const int num_procs = Mpi::WorldSize();
+   if (proc_grid_str == "" || proc_grid_str == "2^n")
+   {
+      if (((num_procs-1)&num_procs) != 0)
+      {
+         if (Mpi::Root())
+         {
+            std::cout << "\nthe number of processors is not a power of 2!"
+                      << " num_procs: " << num_procs
+                      << "\nuse a processor grid, e.g. "
+                      << "--benchmark_context=proc_grid=3x5x7\n"
+                      << std::endl;
+         }
+         return 1;
+      }
+      MakeExp2ProcGrid(num_procs);
+   }
+   else
+   {
+      int n = std::sscanf(proc_grid_str.c_str(), "%d x %d x %d",
+                          &proc_grid[0], &proc_grid[1], &proc_grid[2]);
+      if (n != 3)
+      {
+         if (Mpi::Root())
+         {
+            std::cout << "\ninvalid processor grid input: "
+                      << proc_grid_str << "\n" << std::endl;
+         }
+         return 1;
+      }
+      if (proc_grid[0]*proc_grid[1]*proc_grid[2] != num_procs ||
+          proc_grid[0] < 1 || proc_grid[1] < 1 || proc_grid[2] < 1)
+      {
+         if (Mpi::Root())
+         {
+            std::cout << "\ninvalid processor grid: " << proc_grid[0] << " x "
+                      << proc_grid[1] << " x " << proc_grid[2] << " != "
+                      << num_procs << '\n' << std::endl;
+         }
+         return 1;
+      }
+   }
+
+   if (bm::ReportUnrecognizedArguments(argc, argv)) { return 1; }
+
+   Device device(device_config.c_str());
+   device.SetGPUAwareMPI(gpu_aware_mpi);
+   if (Mpi::Root())
+   {
+      device.Print();
+      std::cout << "    num procs: " << num_procs << std::endl;
+      std::cout << "gpu aware mpi: " << gpu_aware_mpi << std::endl;
+      std::cout << "   local size: " << local_size << std::endl;
+      std::cout << "    proc grid: " << proc_grid[0] << 'x'
+                /**/                 << proc_grid[1] << 'x'
+                /**/                 << proc_grid[2] << std::endl;
+      std::cout << "     GL q_inc: " << q_gl_inc << std::endl;
+      std::cout << "    GLL q_inc: " << q_gll_inc << std::endl;
+   }
+
+   DeviceSync();
+   MPI_Barrier(MPI_COMM_WORLD);
+
+   if (Mpi::Root())
+   {
+      bm::ConsoleReporter CR;
+      bm::RunSpecifiedBenchmarks(&CR);
+   }
+   else
+   {
+      NoReporter NR;
+      bm::RunSpecifiedBenchmarks(&NR);
+   }
+
+   return 0;
+}
+
+#endif // MFEM_USE_BENCHMARK

tests/unit/CMakeLists.txt

@@ -179,25 +179,42 @@ if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
 endif()
 
 #-----------------------------------------------------------
-# SERIAL CUDA TESTS: cunit_tests
+# SERIAL CUDA TESTS: gpu_unit_tests
 #-----------------------------------------------------------
-# Create CUDA 'cunit_tests' executable and test
+# Create CUDA executable and test
 if (MFEM_USE_CUDA)
-   set(CUNIT_TESTS_SRCS cunit_test_main.cpp)
-   set_property(SOURCE ${CUNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-   mfem_add_executable(cunit_tests ${CUNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
-   target_link_libraries(cunit_tests mfem)
-   add_dependencies(cunit_tests copy_data)
-   add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} cunit_tests)
+   # gpu_unit_tests
+   set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
+   set_property(SOURCE ${GPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+   mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+   target_link_libraries(gpu_unit_tests mfem)
+   add_dependencies(gpu_unit_tests copy_data)
+   add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
    if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-      add_test(NAME cunit_tests COMMAND cunit_tests)
+      add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
+   endif()
+endif()
+
+#-----------------------------------------------------------
+# SERIAL HIP TESTS: gpu_unit_tests
+#-----------------------------------------------------------
+# Create HIP 'gpu_unit_tests' executable and test
+if (MFEM_USE_HIP)
+   # gpu_unit_tests
+   set(GPU_UNIT_TESTS_SRCS gpu_unit_test_main.cpp)
+   mfem_add_executable(gpu_unit_tests ${GPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+   target_link_libraries(gpu_unit_tests mfem)
+   add_dependencies(gpu_unit_tests copy_data)
+   add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} gpu_unit_tests)
+   if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+      add_test(NAME gpu_unit_tests COMMAND gpu_unit_tests)
    endif()
 endif()
 
 #-----------------------------------------------------------
 # SERIAL SEDOV + TMOP TESTS:
-#   sedov_tests_{cpu,debug,cuda,cuda_uvm}
-#   tmop_pa_tests_{cpu,debug,cuda}
+#   sedov_tests_{cpu,debug,gpu,gpu_uvm}
+#   tmop_pa_tests_{cpu,debug,gpu}
 #-----------------------------------------------------------
 # Function to add one device serial test from the tests/unit/miniapp directory.
 # All device unit tests are built into a separate executable, in order to be
@@ -226,27 +243,27 @@ function(add_serial_miniapp_test name test_uvm)
         add_test(NAME ${name}_tests_debug COMMAND ${name}_tests_debug)
     endif()
 
-    if (MFEM_USE_CUDA)
-       mfem_add_executable(${name}_tests_cuda ${${NAME}_TESTS_SRCS})
-       target_compile_definitions(${name}_tests_cuda PUBLIC MFEM_${NAME}_DEVICE="cuda")
-       target_link_libraries(${name}_tests_cuda mfem)
-       add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_cuda)
+    if (MFEM_USE_CUDA OR MFEM_USE_HIP)
+       mfem_add_executable(${name}_tests_gpu ${${NAME}_TESTS_SRCS})
+       target_compile_definitions(${name}_tests_gpu PUBLIC MFEM_${NAME}_DEVICE="gpu")
+       target_link_libraries(${name}_tests_gpu mfem)
+       add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} ${name}_tests_gpu)
        if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-           add_test(NAME ${name}_tests_cuda COMMAND ${name}_tests_cuda)
+           add_test(NAME ${name}_tests_gpu COMMAND ${name}_tests_gpu)
        endif()
 
        if (test_uvm)
-           mfem_add_executable(${name}_tests_cuda_uvm ${${NAME}_TESTS_SRCS})
-           target_compile_definitions(${name}_tests_cuda_uvm PUBLIC
-               MFEM_${NAME}_DEVICE="cuda:uvm")
-           target_link_libraries(${name}_tests_cuda_uvm mfem)
+           mfem_add_executable(${name}_tests_gpu_uvm ${${NAME}_TESTS_SRCS})
+           target_compile_definitions(${name}_tests_gpu_uvm PUBLIC
+               MFEM_${NAME}_DEVICE="gpu:uvm")
+           target_link_libraries(${name}_tests_gpu_uvm mfem)
            add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME}
-               ${name}_tests_cuda_uvm)
+               ${name}_tests_gpu_uvm)
            if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-               add_test(NAME ${name}_tests_cuda_uvm COMMAND ${name}_tests_cuda_uvm)
+               add_test(NAME ${name}_tests_gpu_uvm COMMAND ${name}_tests_gpu_uvm)
            endif()
        endif()
-    endif(MFEM_USE_CUDA)
+    endif()
 endfunction(add_serial_miniapp_test)
 
 add_serial_miniapp_test(sedov ON)  # UVM ON
@@ -282,10 +299,11 @@ if (MFEM_USE_CEED)
 endif()
 
 #-----------------------------------------------------------
-# PARALLEL CPU AND CUDA TESTS: {p,pc}unit_tests
+# PARALLEL CPU AND CUDA TESTS: {p,pc}unit_tests and pgpu_unit_tests
 #-----------------------------------------------------------
-# Define executables and tests 'punit_tests' and 'pcunit_tests'
+# Define executables and tests
 if (MFEM_USE_MPI)
+   # punit_tests
    if (MFEM_USE_CUDA)
       set_property(SOURCE punit_test_main.cpp PROPERTY LANGUAGE CUDA)
    endif()
@@ -301,27 +319,44 @@ if (MFEM_USE_MPI)
       endif()
    endforeach()
    if (MFEM_USE_CUDA)
-      set(PCUNIT_TESTS_SRCS pcunit_test_main.cpp)
-      set_property(SOURCE ${PCUNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
-      mfem_add_executable(pcunit_tests ${PCUNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
-      add_dependencies(pcunit_tests copy_data)
-      target_link_libraries(pcunit_tests mfem)
-      add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pcunit_tests)
-      foreach(np 1 ${MFEM_MPI_NP})
-         if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-            add_test(NAME pcunit_tests_np=${np}
-                     COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
-                     ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pcunit_tests>
-                     ${MPIEXEC_POSTFLAGS})
-         endif()
-      endforeach()
+       # pgpu_unit_tests
+       set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
+       set_property(SOURCE ${PGPU_UNIT_TESTS_SRCS} PROPERTY LANGUAGE CUDA)
+       mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+       add_dependencies(pgpu_unit_tests copy_data)
+       target_link_libraries(pgpu_unit_tests mfem)
+       add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
+       foreach(np 1 ${MFEM_MPI_NP})
+          if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+             add_test(NAME pgpu_unit_tests_np=${np}
+                      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
+                      ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pgpu_unit_tests>
+                      ${MPIEXEC_POSTFLAGS})
+          endif()
+       endforeach()
+    endif()
+    if (MFEM_USE_HIP)
+       # pgpu_unit_tests
+       set(PGPU_UNIT_TESTS_SRCS pgpu_unit_test_main.cpp)
+       mfem_add_executable(pgpu_unit_tests ${PGPU_UNIT_TESTS_SRCS} ${UNIT_TESTS_SRCS})
+       add_dependencies(pgpu_unit_tests copy_data)
+       target_link_libraries(pgpu_unit_tests mfem)
+       add_dependencies(${MFEM_ALL_TESTS_TARGET_NAME} pgpu_unit_tests)
+       foreach(np 1 ${MFEM_MPI_NP})
+          if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
+             add_test(NAME pgpu_unit_tests_np=${np}
+                      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${np}
+                      ${MPIEXEC_PREFLAGS} $<TARGET_FILE:pgpu_unit_tests>
+                      ${MPIEXEC_POSTFLAGS})
+          endif()
+       endforeach()
    endif()
 endif(MFEM_USE_MPI)
 
 #-----------------------------------------------------------
 # PARALLEL SEDOV + TMOP TESTS:
-#   psedov_tests_{cpu,debug,cuda,cuda_uvm}
-#   ptmop_pa_tests_{cpu,cuda}
+#   psedov_tests_{cpu,debug,gpu,gpu_uvm}
+#   ptmop_pa_tests_{cpu,gpu}
 #-----------------------------------------------------------
 # Function to add one MPI executable for a test.
 function(add_mpi_executable_test name dev)
@@ -371,10 +406,10 @@ function(add_parallel_miniapp_test name HYPRE_MM)
             list(APPEND backends debug)
         endif()
     endif()
-    if (MFEM_USE_CUDA)
-        list(APPEND backends cuda)
+    if (MFEM_USE_CUDA OR MFEM_USE_HIP)
+        list(APPEND backends gpu)
         if (HYPRE_MM)
-            list(APPEND backends cuda_uvm)
+            list(APPEND backends gpu_uvm)
         endif()
     endif()