Merge branch 'master' into dfem-lvector-interface

Update Xcode version in macos CI from 15.3 -> 16.4

.github/actions/sanitize/config/action.yml

@@ -0,0 +1,154 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: Sanitizer Config
+description: Sets up environment variables for MFEM sanitizer workflow
+
+inputs:
+  DEBUG:
+    description: If true, use intermediate caches to speed up the workflow
+                 by reusing previous builds.
+    default: false
+
+  REPOSITORY:
+    description: Repository to checkout
+    default: mfem/mfem
+
+  BRANCH:
+    description: Branch to checkout
+    default: ubsan
+
+  CLANG_VER:
+    description: CLANG version to use
+    default: 18
+
+  # https://github.com/llvm/llvm-project/releases
+  LLVM_VER:
+    description: LLVM version to use
+    default: 19.1.7
+
+  # https://github.com/hypre-space/hypre/releases
+  HYPRE_VER:
+    description: HYPRE version to use
+    default: 2.19.0
+
+  METIS_VER:
+    description: METIS version to use
+    default: 4.0.3
+
+  CTEST:
+    description: CTest command to use
+    default: ctest -j --test-load $(nproc)
+                      --schedule-random
+                      --stop-on-failure --output-on-failure
+                      --test-dir
+
+  # https://clang.llvm.org/docs/AddressSanitizer.html
+  ASAN_OPTIONS:
+    default: detect_leaks=1,
+             strict_init_order=1,
+             strict_string_checks=1,
+             check_initialization_order=1,
+             detect_stack_use_after_return=1
+  ASAN_CXXFLAGS:
+    default: -fsanitize=address
+             -fsanitize-address-use-after-scope
+  ASAN_LDFLAGS:
+    default: -fsanitize=address
+
+  # https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
+  UBSAN_OPTIONS:
+    default: halt_on_error=1, print_stacktrace=1
+  UBSAN_CXXFLAGS:
+    default: -fsanitize=undefined
+  UBSAN_LDFLAGS:
+    default: -fsanitize=undefined
+
+  # https://clang.llvm.org/docs/MemorySanitizer.html
+  MSAN_OPTIONS:
+    default: "poison_in_dtor=1"
+  MSAN_CXXFLAGS:
+    default: -fsanitize=memory
+             -fsanitize-memory-track-origins
+             -fsanitize-memory-use-after-dtor
+  MSAN_LDFLAGS:
+    default: -fsanitize=memory
+
+  LSAN_DIR:
+    description: LSAN suppression directory
+    default: lsan
+
+  LSAN_FILE:
+    description: LSAN suppression file
+    default: lsan.supp
+
+  NO_FLAGS:
+    description: If true, do not set any CXXFLAGS or LDFLAGS.
+    default: false
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Env (Inputs)
+      run: |
+        echo DEBUG=${{inputs.DEBUG}} >> $GITHUB_ENV
+        echo REPOSITORY=${{inputs.REPOSITORY}} >> $GITHUB_ENV
+        echo BRANCH=${{inputs.BRANCH}} >> $GITHUB_ENV
+        echo CLANG_VER=${{inputs.CLANG_VER}} >> $GITHUB_ENV
+        echo LLVM_VER=${{inputs.LLVM_VER}} >> $GITHUB_ENV
+        echo HYPRE_VER=${{inputs.HYPRE_VER}} >> $GITHUB_ENV
+        echo METIS_VER=${{inputs.METIS_VER}} >> $GITHUB_ENV
+        echo CTEST=${{inputs.CTEST}} >> $GITHUB_ENV
+        echo ASAN_OPTIONS=${{inputs.ASAN_OPTIONS}} >> $GITHUB_ENV
+        echo UBSAN_OPTIONS=${{inputs.UBSAN_OPTIONS}} >> $GITHUB_ENV
+        echo MSAN_OPTIONS=${{inputs.MSAN_OPTIONS}} >> $GITHUB_ENV
+        echo LSAN_DIR=${{inputs.LSAN_DIR}} >> $GITHUB_ENV
+        echo LSAN_FILE=${{inputs.LSAN_FILE}} >> $GITHUB_ENV
+        echo ASAN_CXXFLAGS=${{inputs.ASAN_CXXFLAGS}} >> $GITHUB_ENV
+        echo ASAN_LDFLAGS=${{inputs.ASAN_LDFLAGS}} >> $GITHUB_ENV
+        echo UBSAN_CXXFLAGS=${{inputs.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
+        echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
+        echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
+        echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (dir)
+      run: |
+        echo LLVM_DIR=${{github.workspace}}/llvm >> $GITHUB_ENV
+        echo HYPRE_DIR=hypre-${{inputs.HYPRE_VER}} >> $GITHUB_ENV
+        echo METIS_DIR=metis-${{inputs.METIS_VER}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (bis)
+      run: |
+        echo CC=clang-${{inputs.CLANG_VER}} >> $GITHUB_ENV
+        echo CXX=clang++-${{inputs.CLANG_VER}} >> $GITHUB_ENV
+        echo LLVM_INC=${{env.LLVM_DIR}}/include/c++/v1 >> $GITHUB_ENV
+        echo LLVM_LIB=${{env.LLVM_DIR}}/lib >> $GITHUB_ENV
+        echo HYPRE_TGZ=v${{inputs.HYPRE_VER}}.tar.gz >> $GITHUB_ENV
+        echo METIS_TGZ=metis-${{inputs.METIS_VER}}.tar.gz >> $GITHUB_ENV
+        LSAN_SUPPRESSIONS="${{github.workspace}}/${{inputs.LSAN_DIR}}/${{inputs.LSAN_FILE}}"
+        echo "LSAN_OPTIONS=suppressions=$LSAN_SUPPRESSIONS" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (ter)
+      if: ${{ inputs.NO_FLAGS != 'true' }}
+      run: |
+        echo LLVM_CXXFLAGS=-stdlib=libc++ -I${{env.LLVM_INC}} -Isystem${{env.LLVM_INC}} >> $GITHUB_ENV
+        echo LLVM_LDFLAGS=-L${{env.LLVM_LIB}} -lc++abi -Wl,-rpath,${{env.LLVM_LIB}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (quater)
+      if: ${{ inputs.NO_FLAGS != 'true' }}
+      run: |
+        echo CXXFLAGS=${{env.LLVM_CXXFLAGS}} >> $GITHUB_ENV
+        echo LDFLAGS=${{env.LLVM_LDFLAGS}} >> $GITHUB_ENV
+      shell: bash

.github/actions/sanitize/mfem/action.yml

@@ -0,0 +1,91 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'MFEM Compilation'
+description: 'MFEM Compilation'
+
+inputs:
+  par:
+    description: 'Whether to build for parallel (true/false)'
+    default: false
+  sanitizer:
+    description: 'Sanitizer to use (asan, msan, ubsan)'
+    default: asan
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: ./.github/actions/sanitize/config
+
+    - uses: actions/cache@v4
+      if: ${{env.DEBUG == 'true'}}
+      id: debug
+      with:
+        path: mfem/build
+        key: build-${{inputs.par}}-${{inputs.sanitizer}}
+
+    - uses: ./.github/actions/sanitize/setup
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      with:
+        par: ${{inputs.par}}
+        sanitizer: ${{inputs.sanitizer}}
+
+    - name: Build with ASAN
+      if: inputs.sanitizer == 'asan'
+      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.ASAN_CXXFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Build with MSAN
+      if: inputs.sanitizer == 'msan'
+      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MSAN_CXXFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - name: Build with UBSAN
+      if: inputs.sanitizer == 'ubsan'
+      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
+      shell: bash
+
+    - uses: mfem/github-actions/build-mfem@v2.5
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      env:
+        CXXFLAGS: ${{env.CXXFLAGS}}
+        LDFLAGS: ${{env.LDFLAGS}}
+      with:
+        mpi: ${{inputs.par == 'false' && 'seq' || 'par'}}
+        mfem-dir: mfem
+        os: ${{runner.os}}
+        library-only: true
+        build-system: cmake
+        hypre-dir: ${{env.HYPRE_DIR}}
+        metis-dir: ${{env.METIS_DIR}}
+        config-options: >-
+          -GNinja
+          -DMPICXX=${{env.CXX}}
+          -DCMAKE_CXX_STANDARD=17
+          -DMFEM_USE_MEMALLOC=OFF
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_VERBOSE_MAKEFILE=ON
+          -DCMAKE_CXX_COMPILER=${{env.CXX}}
+          -DCMAKE_CXX_FLAGS_RELEASE='-g -O1 -fno-omit-frame-pointer'
+
+    - name: Delete object files
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      working-directory: mfem/build
+      run: find . -type f -name '*.o' -delete
+      shell: bash
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: build-${{inputs.par}}-${{inputs.sanitizer}}
+        path: mfem/build
+        if-no-files-found: error
+        retention-days: 1
+        overwrite: false

.github/actions/sanitize/mpi/action.yml

@@ -0,0 +1,33 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'Install MPI'
+description: 'Installs MPI and set up its environment variables'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install
+      run: sudo apt-get install openmpi-bin libopenmpi-dev
+      shell: bash
+
+    - name: Env
+      run: |
+        echo PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe >> $GITHUB_ENV
+        echo MPI_INC=$(mpicxx --showme:compile) >> $GITHUB_ENV
+        echo MPI_LIB=$(mpicxx --showme:link) >> $GITHUB_ENV
+      shell: bash
+
+    - name: Env (bis)
+      run: |
+        echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
+        echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV
+      shell: bash

.github/actions/sanitize/restore/action.yml

@@ -0,0 +1,71 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'Restore state'
+description: 'Restore state to be able to run checks, tests'
+
+inputs:
+  par:
+    description: 'Whether to build for parallel (true/false)'
+    default: false
+  sanitizer:
+    description: 'Sanitizer to use (asan, msan, ubsan)'
+    default: asan
+  cache-path:
+    description: 'path to what needs to be restored'
+    default: none
+  cache-skip:
+    description: 'Skip cache restoration'
+    default: false
+
+outputs:
+  cache-hit:
+    description: 'Output from a specific step'
+    value: ${{steps.debug.outputs.cache-hit}}
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: ./.github/actions/sanitize/config
+
+    - uses: actions/cache@v4
+      if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
+      id: debug
+      with:
+        path: ${{inputs.cache-path}}
+        key: ${{github.job}}-${{inputs.par}}-${{inputs.sanitizer}}
+
+    - uses: ./.github/actions/sanitize/setup
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      with:
+        par: ${{inputs.par}}
+        sanitizer: ${{inputs.sanitizer}}
+
+    - uses: actions/download-artifact@v4
+      with:
+        name: build-${{inputs.par}}-${{inputs.sanitizer}}
+        path: mfem/build
+
+    - name: Ninja Patch
+      working-directory: mfem/build
+      run: |
+        sed -i -e 's/CXX_STATIC_LIBRARY_LINKER__mfem_Release.*/CUSTOM_COMMAND/' build.ninja
+        sed -i -e '/build tests\/unit\/all:/ s/tests\/unit\/[^ ]*unit_tests[^ ]*//g' build.ninja
+        sed -i -e '/^add_test(\[=\[\(unit_tests\|punit_tests\)\]=\]/ s/)/ "--input-file .\/list-test-names-${{matrix.tag}}" "--min-duration 1")/' tests/unit/CTestTestfile.cmake
+      shell: bash
+
+    - name: Copy Data
+      if: ${{steps.debug.outputs.cache-hit != 'true'}}
+      working-directory: mfem/build
+      run: |
+        ninja cmake_object_order_depends_target_unit_tests
+        cp -pR ../tests/unit/data tests/unit
+      shell: bash

.github/actions/sanitize/setup/action.yml

@@ -0,0 +1,64 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: 'Setup state'
+description: 'Sets up the state to be able to run build & run'
+
+inputs:
+  par:
+    description: 'Whether to build for parallel (true/false)'
+    default: false
+  sanitizer:
+    description: 'Sanitizer to use (asan, msan, ubsan)'
+    default: asan
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/cache/restore@v4  # Cache for LLVM libcxx
+      with:
+        path: ${{env.LLVM_DIR}}
+        fail-on-cache-miss: true
+        key: build-libcxx-${{env.LLVM_VER}}-${{inputs.sanitizer}}
+
+    - uses: ./.github/actions/sanitize/mpi
+      if: ${{inputs.par == 'true'}}
+
+    - uses: actions/cache/restore@v4  # Cache for Hypre
+      if: ${{inputs.par == 'true'}}
+      with:
+        path: ${{env.HYPRE_DIR}}
+        fail-on-cache-miss: true
+        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+
+    - uses: actions/cache/restore@v4  # Cache for Metis
+      if: ${{inputs.par == 'true'}}
+      with:
+        path: ${{env.METIS_DIR}}
+        fail-on-cache-miss: true
+        key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+
+    - name: Hypre/Metis links
+      if: ${{inputs.par == 'true'}}
+      run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
+      shell: bash
+
+    - uses: actions/cache/restore@v4  # Cache for LSAN suppression file
+      with:
+        path: ${{env.LSAN_DIR}}
+        fail-on-cache-miss: true
+        key: build-lsan-suppression-file
+
+    - uses: actions/checkout@v4  # Checkout the repository
+      with:
+        path: mfem
+        # ref: ${{env.BRANCH}}
+        # repository: ${{env.REPOSITORY}}

.github/workflows/README.md

@@ -7,18 +7,17 @@
 
                                https://mfem.org
 
-
 This directory contains the GitHub CI scripts for MFEM.
 
 Note that some of these scripts use the shared MFEM GitHub Actions from the external mfem/github-actions repository:
 
-  https://github.com/mfem/github-actions
+  <https://github.com/mfem/github-actions>
 
-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.1`, the `v2.1` suffix denotes the branch in the above from which the action is taken.
+For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
 
 The current CI workflows are:
 
-### `repo-check.yml`
+## `repo-check.yml`
 
 Runs a number of static repository-level sanity checks.
 
@@ -30,19 +29,39 @@ Runs a number of static repository-level sanity checks.
 
 - `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
 
-### `mfem-analysis.yml` (`build-analysis`)
+## `mfem-analysis.yml` (`build-analysis`)
 
 Checks if the code builds and satisfies minimal requirements.
 
 - `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
 
-### `builds-and-tests.yml`
+## `builds-and-tests.yml`
 
 Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
 
-Uses the following GitHub Actions from https://github.com/mfem/github-actions:
+Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
 
 - `mfem/github-actions/build-hypre`
 - `mfem/github-actions/build-metis`
 - `mfem/github-actions/build-mfem`
 - `mfem/github-actions/upload-coverage`
+
+## Sanitizer Workflow for MFEM Verification
+
+This workflow validates MFEM unit tests, examples, and miniapps using sanitizer tools.
+
+- `sanitizers.yml` orchestrates:
+  - Building and caching dependencies: HYPRE, METIS, LSAN suppression file, and LLVM libcxx.
+  - Launching fine-grained jobs for serial (ASAN, MSAN, UBSAN) and parallel (ASAN, UBSAN) sanitizers.
+- `sanitize-tests.yml` is a reusable workflow accepting `par` mode (`true` for parallel) and `sanitizer` (ASAN, MSAN, or UBSAN) as inputs. It executes the following jobs:
+  - **Build**: Compiles the MFEM library with specified parallel and sanitizer settings.
+  - **Check**: Runs verification checks.
+  - Parallel jobs to test the following: **Examples**, **Miniapps** and **Unit tests**
+
+The workflow leverages composite actions in `.github/actions/sanitize/`:
+
+- `config`: Centralizes settings for the sanitizer workflow.
+- `mfem`: Manages the MFEM library build process.
+- `mpi`: Installs MPI and applies additional compilation flags.
+- `restore`: Restores the testing environment state.
+- `setup`: Builds or restores cached dependencies.

.github/workflows/builds-and-tests.yml

@@ -168,10 +168,13 @@ jobs:
           env
         shell: bash
 
+      # For info on Xcode see:
+      #  - https://github.com/actions/runner-images/issues/12541
+      #  - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
       - name: Xcode version setup (MacOS)
         if: matrix.os == 'macos-latest'
         run: |
-          XCODE_PATH="/Applications/Xcode_15.3.app"
+          XCODE_PATH="/Applications/Xcode_16.4.app"
           echo "> sudo xcode-select -s ${XCODE_PATH}"
           sudo xcode-select -s ${XCODE_PATH}
           echo "> g++ -v"
@@ -289,10 +292,10 @@ jobs:
         run: |
           export HOMEBREW_NO_INSTALL_CLEANUP=1
           brew update
-          brew install llvm@19 enzyme
-          echo "LLVM_PREFIX=$(brew --prefix llvm@19)" >> $GITHUB_ENV
-          echo "OMPI_CC=$(brew --prefix llvm@19)/bin/clang" >> $GITHUB_ENV
-          echo "OMPI_CXX=$(brew --prefix llvm@19)/bin/clang++" >> $GITHUB_ENV
+          brew install llvm@20 enzyme
+          echo "LLVM_PREFIX=$(brew --prefix llvm@20)" >> $GITHUB_ENV
+          echo "OMPI_CC=$(brew --prefix llvm@20)/bin/clang" >> $GITHUB_ENV
+          echo "OMPI_CXX=$(brew --prefix llvm@20)/bin/clang++" >> $GITHUB_ENV
 
       # MFEM build and test
       - name: build

.github/workflows/mfem-sanitizer.yml

@@ -1,69 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Sanitizer"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  Serial:
-    runs-on: ubuntu-24.04
-
-    steps:
-      - name: MFEM Checkout
-        uses: actions/checkout@v4
-        with:
-          path: mfem
-
-      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.5
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          mpi: seq
-          hypre-dir: unused-hypre-dir
-          metis-dir: unused-metis-dir
-          mfem-dir: mfem
-          build-system: make
-          library-only: false
-          config-options:
-            CXX="clang++-18"
-            CXXFLAGS="-g -O1 -std=c++17
-                      -fsanitize=address
-                      -fno-omit-frame-pointer
-                      -fsanitize-address-use-after-scope"
-
-      - name: MFEM Info
-        working-directory: mfem
-        run: make info
-
-      - name: MFEM Sanitize
-        working-directory: mfem
-        run:
-          ASAN_OPTIONS="detect_leaks=1,
-                        strict_init_order=1,
-                        strict_string_checks=1,
-                        check_initialization_order=1,
-                        detect_stack_use_after_return=1"
-          make test

.github/workflows/sanitize-build-hypre.yml

@@ -0,0 +1,39 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-hypre
+on:
+  workflow_call:
+jobs:
+  build-hypre:
+    runs-on: ubuntu-latest
+    name: 2.19.0
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.HYPRE_DIR}}
+          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
+      - name: Setup
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/sanitize/mpi
+      - name: Build
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-hypre@v2.5
+        with:
+          archive: ${{env.HYPRE_TGZ}}
+          dir: ${{env.HYPRE_DIR}}
+          target: int32
+          precision: fp64
+          build-system: make

.github/workflows/sanitize-build-libcxx.yml

@@ -0,0 +1,76 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-libcxx
+on:
+  workflow_call:
+jobs:
+  build-llvm-libcxx:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        sanitizer: [asan, msan, ubsan]
+        include:
+          - sanitizer: asan
+            llvm_use_sanitizer: "Address"
+          - sanitizer: msan
+            llvm_use_sanitizer: "MemoryWithOrigins"
+          - sanitizer: ubsan
+            llvm_use_sanitizer: "Undefined"
+    name: ${{matrix.sanitizer}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+        with:
+          NO_FLAGS: true
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.LLVM_DIR}}
+          key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}
+      - name: Clone
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: >
+          git clone --filter=blob:none --depth=1
+          --branch llvmorg-${{env.LLVM_VER}}
+          --no-checkout https://github.com/llvm/llvm-project.git llvm-project
+      - name: Checkout
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        working-directory: llvm-project
+        run: |
+          git sparse-checkout set --cone
+          git checkout llvmorg-${{env.LLVM_VER}}
+          git sparse-checkout set cmake llvm/cmake runtimes libcxx libcxxabi
+      - name: Mkdir
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: mkdir ${{env.LLVM_DIR}}
+      - name: CMake
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        working-directory: ${{env.LLVM_DIR}}
+        run: >
+          VERBOSE=1
+          cmake -GNinja ../llvm-project/runtimes/
+          -DCMAKE_C_COMPILER=${{env.CC}}
+          -DCMAKE_CXX_COMPILER=${{env.CXX}}
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo
+          -DCMAKE_INSTALL_PREFIX=/usr
+          -DLLVM_USE_SANITIZER=${{matrix.llvm_use_sanitizer}}
+          -DLLVM_BUILD_32_BITS=OFF
+          -DLIBCXXABI_USE_LLVM_UNWINDER=OFF
+          -DLLVM_INCLUDE_TESTS=OFF
+          -DLIBCXX_INCLUDE_TESTS=OFF
+          -DLIBCXX_INCLUDE_BENCHMARKS=OFF
+          -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi'
+      - name: Build
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        working-directory: ${{env.LLVM_DIR}}
+        run: cmake --build . -- cxx cxxabi

.github/workflows/sanitize-build-lsan.yml

@@ -0,0 +1,38 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-file-lsan
+on:
+  workflow_call:
+jobs:
+  build-file-lsan:
+    runs-on: ubuntu-latest
+    name: lsan.supp
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.LSAN_DIR}}
+          key: build-lsan-suppression-file
+      - name: Setup
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p ${{env.LSAN_DIR}}
+          cat << EOF > ${{env.LSAN_DIR}}/${{env.LSAN_FILE}}
+          leak:libevent_core-2.1.so
+          leak:ompi_mpi_finalize
+          leak:ompi_mpi_init
+          leak:PMPI_Init
+          leak:strdup
+          EOF

.github/workflows/sanitize-build-metis.yml

@@ -0,0 +1,36 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: build-metis
+on:
+  workflow_call:
+jobs:
+  build-metis:
+    runs-on: ubuntu-latest
+    name: 4.0.3
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/config
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ${{env.METIS_DIR}}
+          key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
+      - name: Setup
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/sanitize/mpi
+      - name: Build
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-metis@v2.5
+        with:
+          archive: ${{env.METIS_TGZ}}
+          dir: ${{env.METIS_DIR}}

.github/workflows/sanitize-tests.yml

@@ -0,0 +1,197 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: Sanitize
+on:
+  workflow_call:
+    inputs:
+      par:
+        description: 'Whether to build for parallel (true/false)'
+        required: false
+        default: false
+        type: boolean
+      sanitizer:
+        description: 'Sanitizer to use (asan, msan, ubsan)'
+        required: true
+        default: asan
+        type: string
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/mfem
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+
+  check:
+    needs: [build]
+    runs-on: ubuntu-latest
+    env:
+      ex: ${{inputs.par && 'ex1p' || 'ex1'}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/examples/${{env.ex}}
+      - name: MFEM Check
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v check
+
+  examples:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      exclude: ${{inputs.par && '-E "_ser"' || ''}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/examples/ex1
+      - name: Build Examples
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v examples
+      - name: Test Examples
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} examples ${{env.exclude}} --show-only
+          ${{env.CTEST}} examples ${{env.exclude}}
+
+  miniapps:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      exclude: ${{inputs.par && '-E "_ser"' || ''}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/miniapps/meshing/minimal-surface
+      - name: Build Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v miniapps
+      - name: Test Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} miniapps ${{env.exclude}} --show-only
+          ${{env.CTEST}} miniapps ${{env.exclude}}
+
+  tests-miniapps:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      run: ${{inputs.par && '-R "_cpu_np"' || ''}}
+      exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/tests/unit/sedov_tests_cpu
+      - name: Build Tests Unit Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v tests/unit/all
+      - name: Run Tests Unit Miniapps
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}} --show-only
+          ${{env.CTEST}} tests/unit -E ${{env.exclude}} ${{env.run}}
+
+  tests-unit-build:
+    needs: [check]
+    runs-on: ubuntu-latest
+    env:
+      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
+      - name: Build Unit Tests
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: ninja -v ${{env.unit_tests}}
+      - name: Delete object files
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build/tests/unit
+        run: find . -type f -name '*.o' -delete
+      - uses: actions/upload-artifact@v4
+        with:
+          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
+          path: mfem/build/tests/unit/${{env.unit_tests}}
+          if-no-files-found: error
+          retention-days: 1
+          overwrite: false
+
+  tests-unit-run:
+    needs: [tests-unit-build]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tag: [0, 1, 2, 3]
+    name: tests-unit-run-${{matrix.tag}}
+    env:
+      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
+      np: ${{inputs.par && '_np=2' || ''}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/sanitize/restore
+        id: restore
+        with:
+          par: ${{inputs.par}}
+          sanitizer: ${{inputs.sanitizer}}
+          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
+      - uses: actions/download-artifact@v4
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        with:
+          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
+          path: mfem/build/tests/unit
+      - name: Split Unit Tests
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build/tests/unit
+        run: |
+          chmod 755 ${{env.unit_tests}}
+          ./${{env.unit_tests}} --list-test-names-only | tail -n +2 > list-test-names
+          shuf list-test-names -o list-test-names
+          split --verbose -n l/4 -d -a 1 list-test-names list-test-names-
+      - name: Cat Unit Tests ${{matrix.tag}}
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build/tests/unit
+        run: cat list-test-names-${{matrix.tag}}
+      - name: Run Unit Tests ${{matrix.tag}}
+        if: ${{steps.restore.outputs.cache-hit != 'true'}}
+        working-directory: mfem/build
+        run: |
+          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}" --show-only
+          ${{env.CTEST}} tests/unit -R "${{env.unit_tests}}${{env.np}}"

.github/workflows/sanitizers.yml

@@ -0,0 +1,73 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+---
+name: Sanitizers
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches: ["master", "next"]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+
+  # Build steps for dependencies
+  build-hypre:
+    uses: ./.github/workflows/sanitize-build-hypre.yml
+
+  build-metis:
+    uses: ./.github/workflows/sanitize-build-metis.yml
+
+  build-lsan:
+    uses: ./.github/workflows/sanitize-build-lsan.yml
+
+  build-libcxx:
+    uses: ./.github/workflows/sanitize-build-libcxx.yml
+
+  # Serial sanitizers: asan, msan, ubsan
+  seq-asan:
+    needs: [build-libcxx]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      sanitizer: asan
+
+  seq-msan:
+    needs: [build-libcxx]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      sanitizer: msan
+
+  seq-ubsan:
+    needs: [build-libcxx]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      sanitizer: ubsan
+
+  # Parallel sanitizers: asan, ubsan
+  par-asan:
+    needs: [build-libcxx, build-hypre, build-metis]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      par: true
+      sanitizer: asan
+  par-ubsan:
+    needs: [build-libcxx, build-hypre, build-metis]
+    uses: ./.github/workflows/sanitize-tests.yml
+    with:
+      par: true
+      sanitizer: ubsan

.gitignore

@@ -211,7 +211,7 @@ miniapps/electromagnetics/joule
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
-miniapps/electromagnetics/Joule_*
+miniapps/electromagnetics/Joule_[0-9]*
 
 miniapps/gslib/field-diff
 miniapps/gslib/field-interp
@@ -267,9 +267,9 @@ miniapps/meshing/bounding-box*
 miniapps/meshing/jacobian-determinant*
 
 miniapps/mtop/parheat
-miniapps/mtop/ParHeat*
+miniapps/mtop/ParHeat/*
 miniapps/mtop/seqheat
-miniapps/mtop/SeqHeat*
+miniapps/mtop/SeqHeat/*
 
 miniapps/autodiff/paradiff
 miniapps/autodiff/seqadiff
@@ -277,7 +277,7 @@ miniapps/autodiff/seqtest
 miniapps/autodiff/par_example
 miniapps/autodiff/seq_example
 miniapps/autodiff/seq_test
-miniapps/autodiff/Exampl*
+miniapps/autodiff/Example/*
 
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
@@ -300,6 +300,7 @@ miniapps/nurbs/nurbs_solenoidal
 miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
+miniapps/nurbs/nurbs_surface
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol_?.gf
@@ -318,6 +319,7 @@ miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
 miniapps/nurbs/glvis_naca-cmesh.mesh
 miniapps/nurbs/Naca_cmesh
+miniapps/nurbs/*-Surface.mesh
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -413,8 +415,8 @@ miniapps/diag-smoothers/mg-abs-l1-jacobi
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/cunit_tests
-tests/unit/pcunit_tests
+tests/unit/gpu_unit_tests
+tests/unit/pgpu_unit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*

.gitlab-ci.yml

@@ -22,7 +22,7 @@ include:
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where ruby resource are allocated/released once for all.
+# - Allocate/Release is where Dane resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
@@ -53,7 +53,7 @@ variables:
   AUTOTEST_COMMIT: "YES"
 
 # Trigger subpipelines:
-ruby-build-and-test:
+dane-build-and-test:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -61,10 +61,10 @@ ruby-build-and-test:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/ruby-build-and-test.yml
+    include: .gitlab/dane-build-and-test.yml
     strategy: depend
 
-ruby-baseline:
+dane-baseline:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -73,7 +73,7 @@ ruby-baseline:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/ruby-baseline.yml
+    include: .gitlab/dane-baseline.yml
     strategy: depend
 
 lassen-build-and-test:

.gitlab/README.md

@@ -24,7 +24,7 @@ and `test type`.
 
 Machines typically include:
 
-* Ruby: 2nd Gen Intel Xeon (Cascade Lake)
+* Dane: Intel Sapphire Rapids
 * Lassen: Power9 + Nvidia GPU
 * Corona: AMD GPU
 
@@ -76,13 +76,13 @@ with a spack spec of MFEM, within the limits permitted by the MFEM spack
 package.
 
 In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on ruby for example resumes to:
+spack spec to use. Adding a job on Dane for example resumes to:
 
 ```yaml
 <job_name>:
   variables:
     SPEC: "<spack_spec>"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 ```
 
 The remaining and non trivial work is to make sure this spec is working. To

.gitlab/configs/common.yml

@@ -24,7 +24,7 @@ variables:
 # TODO: add a clean-up mechanism
   BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
 
-# On LLNL's ruby, there is only one allocation shared among jobs in order to
+# On LLNL's Dane, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}

.gitlab/configs/dane-config.yml

@@ -9,17 +9,17 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# GitLab pipelines configurations for the Ruby machine at LLNL
+# GitLab pipelines configurations for the Dane machine at LLNL
 variables:
-  MACHINE_NAME: ruby
+  MACHINE_NAME: dane
 
-.on_ruby:
+.on_dane:
   tags:
     - shell
-    - ruby
+    - dane
   rules:
-    # Don't run ruby jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"'
+    # Don't run dane jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"'
       when: never
     # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
@@ -40,16 +40,17 @@ variables:
     - when: on_success
 
 # Spack helped builds
-# Generic ruby build job, extending build script
-.build_and_test_on_ruby:
-  extends: [.on_ruby]
+# Generic dane build job, extending build script
+.build_and_test_on_dane:
+  extends: [.on_dane]
   stage: build_and_test
   script:
     # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=16
+    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
+    - export THREADS=28
     - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
     - echo ${JOBID}
     - echo ${MFEM_DATA_DIR}
     - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/setup-baseline.yml

@@ -18,7 +18,7 @@
 setup_baseline:
   tags:
     - shell
-    - ruby
+    - dane
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/configs/setup-build-and-test.yml

@@ -16,7 +16,7 @@
 setup:
   tags:
     - shell
-    - ruby
+    - dane
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/dane-baseline.yml

@@ -19,8 +19,8 @@ stages:
   - cleanup
   - baseline_publish
 
-baselinecheck_mfem_intel_ruby:
-  extends: [.on_ruby]
+baselinecheck_mfem_intel_dane:
+  extends: [.on_dane]
   stage: baseline_check
   variables:
     # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
@@ -31,8 +31,8 @@ baselinecheck_mfem_intel_ruby:
   script:
     - echo ${BUILD_ROOT}
     - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=48
+    # Used by the tests in MFEM/tests, dane has 224 threads/node:
+    - export MFEM_TEST_NP=192
     # The next script uses the following environment variables:
     # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
     # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
@@ -44,7 +44,7 @@ baselinecheck_mfem_intel_ruby:
   allow_failure: true
 
 cleanup:
-  extends: .on_ruby
+  extends: .on_dane
   stage: cleanup
   variables:
     GIT_STRATEGY: none
@@ -53,7 +53,7 @@ cleanup:
     - rm -rf "${BUILD_ROOT}" || true
 
 report_baseline:
-  extends: [.on_ruby]
+  extends: [.on_dane]
   stage: baseline_report
   script:
     - echo ${MACHINE_NAME}
@@ -113,8 +113,8 @@ report_baseline:
         exit $err
       ) 9> autotest.lock
 
-baselinepublish_mfem_ruby:
-  extends: [.on_ruby]
+baselinepublish_mfem_dane:
+  extends: [.on_dane]
   stage: baseline_publish
   rules:
     # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
@@ -129,5 +129,5 @@ baselinepublish_mfem_ruby:
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/dane-config.yml
   - local: .gitlab/configs/setup-baseline.yml

.gitlab/dane-build-and-test.yml

@@ -19,54 +19,54 @@ stages:
 allocate_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_dane
   stage: allocate_resource
   script:
     - echo ${ALLOC_NAME}
     - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
   timeout: 6h
 
-# GitLab jobs for the Ruby machine at LLNL
+# GitLab jobs for the Dane machine at LLNL
 debug_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 
 debug_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 
 opt_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 
 opt_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 
 opt_par_gcc_10_sundials:
   variables:
     SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 
 opt_par_gcc_10_petsc:
   variables:
     SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 
 opt_par_gcc_10_pumi:
   variables:
     SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_ruby
+  extends: .build_and_test_on_dane
 
 # Release
 release_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_ruby
+  extends: .on_dane
   stage: release_resource_and_report
   script:
     - echo ${ALLOC_NAME}
@@ -78,17 +78,17 @@ release_resource:
 report_job_success:
   stage: release_resource_and_report
   extends:
-    - .on_ruby
+    - .on_dane
     - .report_job_success
 
 report_job_failure:
   stage: release_resource_and_report
   extends:
-    - .on_ruby
+    - .on_dane
     - .report_job_failure
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/ruby-config.yml
+  - local: .gitlab/configs/dane-config.yml
   - local: .gitlab/configs/setup-build-and-test.yml
   - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -14,7 +14,7 @@
 # locals
 glob_err=${BASELINE_TEST}.err
 base=${BASELINE_TEST}-${SYS_TYPE}
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
+if [[ "${MACHINE_NAME}" == "dane" ]]; then
   base="${BASELINE_TEST}-${MACHINE_NAME}"
 fi
 base_diff=${base}.diff
@@ -31,7 +31,7 @@ cd tests
 mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
-if [[ "${MACHINE_NAME}" == "ruby" ]]; then
+if [[ "${MACHINE_NAME}" == "dane" ]]; then
   salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
   salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"

.gitlab/scripts/rebaseline

@@ -11,7 +11,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and dane baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
@@ -21,7 +21,7 @@ PATCH_FILE=${ARTIFACT_PATH}.patch
 FULL_FILE=${ARTIFACT_PATH}.out
 DIFF_FILE=${ARTIFACT_PATH}.diff
 
-# There will be collision between corona and ruby baselines.
+# There will be collision between corona and dane baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 SAVED_NAME=baseline-${SYS_TYPE}.saved

CHANGELOG

@@ -29,9 +29,14 @@ Discretization improvements
 
 Meshing improvements
 --------------------
+
 - Added support for higher order meshes in Mesh::MakeSimplicial and
   ParMesh::MakeSimplicial.
 
+- Added a new miniapp for interpolating a surface grid of points in 3D using a
+  smooth NURBS surface, that can then be sampled at arbitrary resolution while
+  staying close to the original geometry. See miniapps/nurbs/nurbs_surface.
+
 GPU computing
 -------------
 - The function Vector::SetSubVector(const Array<int> &, const real_t) now
@@ -39,6 +44,13 @@ GPU computing
   set. This is most often used for setting constant essential boundary
   conditions. A new function Vector::SetSubVectorHost has been added in cases
   where host execution is always needed (e.g. when the DOFs array is small).
+- Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
+  threads, assigning one task per thread.
+- Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction`
+  update operator. This supports mixed geometry meshes and variable order
+  spaces, and is the default derefinement operator constructed by
+  `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`.
+  The operator requires `FiniteElementSpace::Nonconforming() == true`.
 
 New and updated examples and miniapps
 -------------------------------------
@@ -48,13 +60,26 @@ New and updated examples and miniapps
   operators as smoothers.
   These miniapps can be found in `miniapps/diag-smoothers`.
 
-API changes:
+API changes
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
   mfem::future::tensor and mfem::future::dual.
 - API addition: in class `Operator`, added virtual functions: `AbsMult`, and
   `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.
 
+Miscellaneous
+-------------
+- Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
+  automatically select between CUDA or HIP.
+- The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
+  'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
+  CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
+  MFEM build configuration.
+- Added the option to enable GPU-aware MPI in MFEM using the environment
+  variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
+  variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
+- Added parallel Address Sanitizer, serial and parallel Undefined Behavior
+  Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
 
 Version 4.8, released on Apr 9, 2025
 ====================================

CMakeLists.txt

@@ -598,14 +598,20 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
     ALGOIM ENZYME)
 
-# Add all *_FOUND libraries in the variable TPL_LIBRARIES.
+# Add all created targets and *_FOUND libraries in the variables TPL_TARGETS and
+# TPL_LIBRARIES, respectively.
+set(TPL_TARGETS)
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
 foreach(TPL IN LISTS MFEM_TPLS)
-  if (${TPL}_FOUND)
+  if (${TPL}_FOUND OR TARGET ${TPL})
     message(STATUS "MFEM: using package ${TPL}")
-    list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
-    list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
+    if (TARGET ${TPL})
+      list(APPEND TPL_TARGETS ${TPL})
+    else()
+      list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
+      list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
+    endif()
   endif()
 endforeach(TPL)
 list(REVERSE TPL_LIBRARIES)
@@ -680,7 +686,10 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
+if (TPL_TARGETS)
+  add_dependencies(mfem ${TPL_TARGETS})
+endif()
 if (MINGW)
   target_link_libraries(mfem PRIVATE ws2_32)
 endif()

INSTALL

@@ -121,6 +121,11 @@ Parallel build:
    make -j 4
 (For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
 
+Parallel build with fetching of hypre and METIS:
+   mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
+   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DFETCH_TPLS=YES
+   make -j 4
+
 CUDA build:
    (this build requires CMake 3.17 or newer)
    mkdir <mfem-build-dir> ; cd <mfem-build-dir>
@@ -663,6 +668,7 @@ The specific libraries and their options are:
 - OpenMP (optional), usually part of compiler, used when either MFEM_USE_OPENMP
   or MFEM_USE_LEGACY_OPENMP is set to YES.
   Options: OPENMP_OPT, OPENMP_LIB.
+  Versions: OpenMP >= 3.1 when MFEM_USE_OPENMP=YES.
 
 - High-resolution POSIX clocks: when using MFEM_TIMER_TYPE = 2, it may be
   necessary to link with a system library (e.g. librt.so).
@@ -842,6 +848,7 @@ The specific libraries and their options are:
 - HIP (optional), used when MFEM_USE_HIP = YES.
   URL: https://rocmdocs.amd.com
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
+  Versions: ROCm >= 5.6.1.
 
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
   URL: https://libocca.org
@@ -1074,6 +1081,9 @@ The following options are CMake specific:
 MFEM_ENABLE_TESTING  - Enable the ctest framework for testing.
 MFEM_ENABLE_EXAMPLES - Build all of the examples by default.
 MFEM_ENABLE_MINIAPPS - Build all of the miniapps by default.
+FETCH_TPLS           - Enable fetching of all supported third-party libraries.
+HYPRE_FETCH          - Enable fetching of hypre.
+METIS_FETCH          - Enable fetching of metis.
 
 External libraries (CMake):
 ---------------------------
@@ -1135,6 +1145,12 @@ The following built-in CMake packages are also used:
       set the <LIBNAME>_LIBRARIES option directly; the configuration option
       <LIBNAME>_DIR is not supported.
 
+The MFEM CMake build system also provides fetching (automated building) for the
+packages/libraries listed below. Note that when fetching is enabled, any related
+auto-detection functionality is disabled.
+
+ - HYPRE
+ - METIS
 
 Building without GNU make or CMake
 ==================================

config/cmake/modules/FindHYPRE.cmake

@@ -9,15 +9,18 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# Defines the following variables:
+# Defines the following variables if fetching of TPLs is disabled (default):
 #   - HYPRE_FOUND
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
 #   - HYPRE_USING_CUDA (internal)
 #   - HYPRE_USING_HIP (internal)
+# otherwise, the following are defined:
+#   - HYPRE (imported library target)
+#   - HYPRE_VERSION (cache variable)
 
-if (HYPRE_FOUND)
+if (HYPRE_FOUND OR TARGET HYPRE)
   if (HYPRE_USING_CUDA)
     find_package(CUDAToolkit REQUIRED)
   endif()
@@ -33,6 +36,55 @@ if (HYPRE_FOUND)
   endif()
 endif()
 
+if (HYPRE_FETCH OR FETCH_TPLS)
+  set(HYPRE_FETCH_VERSION 2.33.0)
+  add_library(HYPRE STATIC IMPORTED)
+  # set options and associated dependencies
+  set(CMAKE_OPTIONS)
+  list(APPEND CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
+  if (MFEM_USE_CUDA)
+    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_CUDA:BOOL=ON)
+    find_package(CUDAToolkit REQUIRED)
+    target_link_libraries(HYPRE INTERFACE CUDA::cusparse CUDA::curand CUDA::cublas)
+  elseif (MFEM_USE_HIP)
+    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_HIP:BOOL=ON)
+    find_package(rocsparse REQUIRED)
+    find_package(rocrand REQUIRED)
+    target_link_libraries(HYPRE INTERFACE rocsparse rocrand)
+  endif()
+  if (MFEM_USE_SINGLE)
+    list(APPEND CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON)
+  endif()
+  # define external project and create future include directory so it is present
+  # to pass CMake checks at end of MFEM configuration step
+  message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_VERSION} to be built with ${CMAKE_OPTIONS}")
+  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/hypre)
+  include(ExternalProject)
+  ExternalProject_Add(hypre
+    GIT_REPOSITORY https://github.com/hypre-space/hypre.git
+    GIT_TAG v${HYPRE_FETCH_VERSION}
+    GIT_SHALLOW TRUE
+    UPDATE_DISCONNECTED TRUE
+    SOURCE_SUBDIR src
+    PREFIX ${PREFIX}
+    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${CMAKE_OPTIONS})
+  file(MAKE_DIRECTORY ${PREFIX}/include)
+  # set imported library target properties
+  add_dependencies(HYPRE hypre)
+  set_target_properties(HYPRE PROPERTIES
+    IMPORTED_LOCATION ${PREFIX}/lib/libHYPRE.a
+    INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include)
+  # convert HYPRE version to integer
+  string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION ${HYPRE_FETCH_VERSION})
+  list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION)
+  list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION)
+  list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION)
+  math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}")
+  # set cache variables that would otherwise be set after mfem_find_package call
+  set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE)
+  return()
+endif()
+
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
   "Paths to headers required by HYPRE." "Libraries required by HYPRE."

config/cmake/modules/FindMETIS.cmake

@@ -9,10 +9,38 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# Defines the following variables:
+# Defines the following variables if fetching of TPLs is disabled (default):
 #   - METIS_FOUND
 #   - METIS_LIBRARIES
 #   - METIS_INCLUDE_DIRS
+#   - METIS_VERSION_5
+# otherwise, the following are defined:
+#   - METIS (imported library target)
+#   - METIS_VERSION_5 (cache variable)
+
+if (METIS_FETCH OR FETCH_TPLS)
+  set(METIS_FETCH_VERSION 4.0.3)
+  add_library(METIS STATIC IMPORTED)
+  # define external project
+  message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with default options")
+  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/metis)
+  include(ExternalProject)
+  ExternalProject_Add(metis
+    GIT_REPOSITORY https://github.com/mfem/tpls
+    GIT_TAG b60352fbe9675d374b00828055e55be4584c7995 # tag from 1/16/25
+    GIT_SHALLOW TRUE
+    UPDATE_DISCONNECTED TRUE
+    PREFIX ${PREFIX}
+    CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
+    INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
+  # set imported library target properties
+  add_dependencies(METIS metis)
+  set_target_properties(METIS PROPERTIES
+    IMPORTED_LOCATION ${PREFIX}/lib/libmetis.a)
+  # set cache variables that would otherwise be set after mfem_find_package call
+  set(METIS_VERSION_5 FALSE CACHE BOOL "Is METIS version 5?")
+  return()
+endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(METIS METIS METIS_DIR "include;Lib" "metis.h"

config/config.hpp

@@ -23,11 +23,14 @@
 #include "_config.hpp"
 #endif
 
+#include <cstdint>
+#include <climits>
+
 namespace mfem
 {
 
 #if (defined(MFEM_USE_CUDA) && defined(__CUDACC__)) || \
-    (defined(MFEM_USE_HIP) && defined(__HIPCC__))
+    (defined(MFEM_USE_HIP) && defined(__HIP__))
 #define MFEM_HOST_DEVICE __host__ __device__
 #else
 #define MFEM_HOST_DEVICE

config/defaults.cmake

@@ -89,6 +89,12 @@ option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 option(MFEM_ENABLE_BENCHMARKS "Build all of the benchmarks" OFF)
 
+# Allow a user to specify fetching of certain third-party libraries instead of
+# searching for existing installations.
+option(FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
+option(HYPRE_FETCH "Enable fetching of hypre" OFF)
+option(METIS_FETCH "Enable fetching of METIS" OFF)
+
 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
 # set(CXX g++)

fem/CMakeLists.txt

@@ -82,6 +82,8 @@ set(SRCS
   fe/fe_ser.cpp
   fe_coll.cpp
   fespace.cpp
+  derefmat_op.cpp
+  pderefmat_op.cpp
   geom.cpp
   gridfunc.cpp
   hybridization.cpp
@@ -247,6 +249,7 @@ set(HDRS
   nonlinearform_ext.hpp
   nonlininteg.hpp
   qfunction.hpp
+  qinterp/det.hpp
   qinterp/eval.hpp
   qinterp/eval_hdiv.hpp
   qinterp/grad.hpp

fem/batchitrans.cpp

@@ -515,6 +515,7 @@ struct InvTNewtonSolver<Geometry::SEGMENT, SDim, SType, max_team_x>
             phys_tol += pptr[idx + d * npts] * pptr[idx + d * npts];
          }
          phys_tol = fmax(phys_rtol * phys_rtol, phys_tol * phys_rtol * phys_rtol);
+         hit_bdr[0] = prev_hit_bdr[0] = false;
       }
       // for each iteration
       while (true)

fem/bilinearform_ext.cpp

@@ -266,11 +266,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 
       // Gather the attributes on the host from all the elements
       const Mesh &mesh = *trial_fes->GetMesh();
-      elem_attributes.SetSize(mesh.GetNE());
-      for (int i = 0; i < mesh.GetNE(); ++i)
-      {
-         elem_attributes[i] = mesh.GetAttribute(i);
-      }
+      elem_attributes = &mesh.GetElementAttributes();
    }
 
    // Construct face restriction operators only if the bilinear form has
@@ -329,45 +325,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
          bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height());
       }
 
-      const Mesh &mesh = *trial_fes->GetMesh();
-      // See LinearFormExtension::Update for explanation of f_to_be logic.
-      std::unordered_map<int,int> f_to_be;
-      for (int i = 0; i < mesh.GetNBE(); ++i)
-      {
-         const int f = mesh.GetBdrElementFaceIndex(i);
-         f_to_be[f] = i;
-      }
-      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
-      bdr_attributes.SetSize(nf_bdr);
-      int f_ind = 0;
-      int missing_bdr_elems = 0;
-      for (int f = 0; f < mesh.GetNumFaces(); ++f)
-      {
-         if (!mesh.GetFaceInformation(f).IsOfFaceType(FaceType::Boundary))
-         {
-            continue;
-         }
-         int attribute = 1; // default value
-         if (f_to_be.find(f) != f_to_be.end())
-         {
-            const int be = f_to_be[f];
-            attribute = mesh.GetBdrAttribute(be);
-         }
-         else
-         {
-            // If a boundary face does not correspond to the a boundary element,
-            // we assign it the default attribute of 1. We also generate a
-            // warning at runtime with the number of such missing elements.
-            ++missing_bdr_elems;
-         }
-         bdr_attributes[f_ind] = attribute;
-         ++f_ind;
-      }
-      if (missing_bdr_elems)
-      {
-         MFEM_WARNING("Missing " << missing_bdr_elems << " boundary elements "
-                      "for boundary faces.");
-      }
+      bdr_face_attributes = &trial_fes->GetMesh()->GetBdrFaceAttributes();
    }
 }
 
@@ -429,7 +387,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
          mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
          {
             const int attr = d_attr[e];
-            if (d_m[attr - 1] == 0)
+            if (attr <= 0 || d_m[attr - 1] == 0)
             {
                for (int i = 0; i < nd; ++i)
                {
@@ -450,7 +408,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
          for (int i = 0; i < iSz; ++i)
          {
             assemble_diagonal_with_markers(*integrators[i], elem_markers[i],
-                                           elem_attributes, localY);
+                                           *elem_attributes, localY);
          }
          const ElementRestriction* H1elem_restrict =
             dynamic_cast<const ElementRestriction*>(elem_restrict);
@@ -476,7 +434,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
       for (int i = 0; i < iSz; ++i)
       {
          assemble_diagonal_with_markers(*integrators[i], elem_markers[i],
-                                        elem_attributes, y);
+                                        *elem_attributes, y);
       }
    }
 
@@ -489,7 +447,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
       for (int i = 0; i < n_bdr_integs; ++i)
       {
          assemble_diagonal_with_markers(*bdr_integs[i], bdr_markers[i],
-                                        bdr_attributes, bdr_face_Y);
+                                        *bdr_face_attributes, bdr_face_Y);
       }
       bdr_face_restrict_lex->AddAbsMultTranspose(bdr_face_Y, y);
    }
@@ -588,7 +546,7 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
          for (int i = 0; i < iSz; ++i)
          {
             AddMultWithMarkers(*integrators[i], localX, elem_markers[i],
-                               elem_attributes, false, localY, useAbs);
+                               *elem_attributes, false, localY, useAbs);
          }
          if (H1elem_restrict && useAbs)
          {
@@ -690,8 +648,8 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
          }
          for (int i = 0; i < n_bdr_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
-                               false, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i],
+                               *bdr_face_attributes, false, bdr_face_Y);
          }
          for (int i = 0; i < n_bdr_face_integs; ++i)
          {
@@ -699,12 +657,14 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
             {
                AddMultNormalDerivativesWithMarkers(
                   *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn,
-                  bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn);
+                  bdr_face_markers[i], *bdr_face_attributes, bdr_face_Y,
+                  bdr_face_dYdn);
             }
             else
             {
-               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                                  bdr_attributes, false, bdr_face_Y);
+               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X,
+                                  bdr_face_markers[i], *bdr_face_attributes, false,
+                                  bdr_face_Y);
             }
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
@@ -727,7 +687,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       localY = 0.0;
       for (int i = 0; i < iSz; ++i)
       {
-         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
+         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], *elem_attributes,
                             true, localY);
       }
       elem_restrict->MultTranspose(localY, y);
@@ -774,13 +734,14 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          bdr_face_Y = 0.0;
          for (int i = 0; i < n_bdr_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
-                               true, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i],
+                               *bdr_face_attributes, true, bdr_face_Y);
          }
          for (int i = 0; i < n_bdr_face_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                               bdr_attributes, true, bdr_face_Y);
+            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X,
+                               bdr_face_markers[i], *bdr_face_attributes, true,
+                               bdr_face_Y);
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
@@ -804,7 +765,7 @@ static void AddWithMarkers_(
    mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
    {
       const int attr = d_attr[e];
-      if (d_m[attr - 1] == 0) { return; }
+      if (attr <= 0 || d_m[attr - 1] == 0) { return; }
       for (int i = 0; i < nd; ++i)
       {
          d_y(i, e) += d_x(i, e);
@@ -920,7 +881,8 @@ void EABilinearFormExtension::Assemble()
       {
          const int i = idx % sz;
          const int e = idx / sz;
-         const real_t val = d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0.0;
+         const real_t val =
+            d_a[e] > 0 ? (d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0) : 0;
          if (add)
          {
             d_ea_2(i, e) += val;
@@ -953,7 +915,7 @@ void EABilinearFormExtension::Assemble()
             ea_data_tmp.SetSize(ea_data.Size());
             integrators[i]->AssembleEA(*a->FESpace(), ea_data_tmp, false);
             add_with_markers(ea_data_tmp, ea_data, ne, *markers,
-                             elem_attributes, add);
+                             *elem_attributes, add);
          }
       }
    }
@@ -982,7 +944,7 @@ void EABilinearFormExtension::Assemble()
             ea_data_tmp.SetSize(ea_data_bdr.Size());
             bdr_integs[i]->AssembleEABoundary(*a->FESpace(), ea_data_tmp, add);
             add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
-                             bdr_attributes, add);
+                             *bdr_face_attributes, add);
          }
       }
    }
@@ -1031,7 +993,7 @@ void EABilinearFormExtension::Assemble()
                                                         ea_data_tmp,
                                                         add);
             add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
-                             bdr_attributes, add);
+                             *bdr_face_attributes, add);
          }
       }
    }

fem/bilinearform_ext.hpp

@@ -69,7 +69,8 @@ class PABilinearFormExtension : public BilinearFormExtension
 protected:
    const FiniteElementSpace *trial_fes, *test_fes; // Not owned
    /// Attributes of all mesh elements.
-   Array<int> elem_attributes, bdr_attributes;
+   const Array<int> *elem_attributes; // Not owned
+   const Array<int> *bdr_face_attributes; // Not owned
    mutable Vector tmp_evec; // Work array
    mutable Vector localX, localY;
    mutable Vector int_face_X, int_face_Y;

fem/bilininteg.hpp

@@ -812,7 +812,7 @@ protected:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetDim() == 1 && test_fe.GetDim() == 1 &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -884,7 +884,7 @@ protected:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV  &&
+      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -919,7 +919,7 @@ protected:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV  &&
+      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 
@@ -1600,7 +1600,7 @@ public:
    {
       return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL   &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 
@@ -1635,7 +1635,7 @@ public:
    {
       return (trial_fe.GetDim() == 2 && test_fe.GetDim() == 2 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL   &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 
@@ -1669,7 +1669,7 @@ public:
    {
       return (trial_fe.GetDim() == 2 && test_fe.GetDim() == 2 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -1760,7 +1760,7 @@ public:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -1793,7 +1793,7 @@ public:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR &&
               test_fe.GetDerivType()  == mfem::FiniteElement::DIV   );
    }
@@ -1832,7 +1832,7 @@ public:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::DIV    &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::DIV   &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR &&
               test_fe.GetDerivType()  == mfem::FiniteElement::GRAD
              );
@@ -1973,7 +1973,7 @@ protected:
       const FiniteElement & test_fe) const override
    {
       return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 

fem/conduitdatacollection.cpp

@@ -912,7 +912,7 @@ ConduitDataCollection::GridFunctionToBlueprintField(mfem::GridFunction *gf,
 
    if (vdim == 1) // scalar case
    {
-      n_field["values"].set_external(gf->GetData(),
+      n_field["values"].set_external(const_cast<real_t *>(gf->HostRead()),
                                      ndofs);
    }
    else // vector case
@@ -925,18 +925,18 @@ ConduitDataCollection::GridFunctionToBlueprintField(mfem::GridFunction *gf,
       int vdim_stride  = (ordering == Ordering::byNODES ? ndofs : 1);
 
       index_t offset = 0;
-      index_t stride = sizeof(double) * entry_stride;
+      index_t stride = sizeof(real_t) * entry_stride;
 
       for (int d = 0;  d < vdim; d++)
       {
          std::ostringstream oss;
          oss << "v" << d;
          std::string comp_name = oss.str();
-         n_field["values"][comp_name].set_external(gf->GetData(),
+         n_field["values"][comp_name].set_external(const_cast<real_t *>(gf->HostRead()),
                                                    ndofs,
                                                    offset,
                                                    stride);
-         offset +=  sizeof(double) * vdim_stride;
+         offset +=  sizeof(real_t) * vdim_stride;
       }
    }
 

fem/datacollection.cpp

@@ -764,9 +764,9 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 {
    cycle = 0;
 #ifdef MFEM_USE_ZLIB
-   compression = true; // if we have zlib, enable compression
-#else
-   compression = false; // otherwise, disable compression
+   // If we have zlib, enable compression. Otherwise, compression is disabled in
+   // the DataCollection base class constructor.
+   compression = true;
 #endif
 }
 
@@ -784,13 +784,8 @@ void ParaViewDataCollectionBase::SetCompressionLevel(int compression_level_)
 {
    MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9,
                "Compression level must be between -1 and 9 (inclusive).");
+   if (compression_level_ != 0) { SetCompression(true);}
    compression_level = compression_level_;
-   compression = compression_level_ != 0;
-}
-
-void ParaViewDataCollectionBase::SetCompression(bool compression_)
-{
-   compression = compression_;
 }
 
 int ParaViewDataCollectionBase::GetCompressionLevel() const
@@ -1174,7 +1169,14 @@ const char *ParaViewDataCollection::GetDataTypeString() const
 ParaViewHDFDataCollection::ParaViewHDFDataCollection(
    const std::string &collection_name, Mesh *mesh)
    : ParaViewDataCollectionBase(collection_name, mesh)
-{ }
+{
+   compression = true;
+}
+
+void ParaViewHDFDataCollection::SetCompression(bool compression_)
+{
+   compression = compression_;
+}
 
 void ParaViewHDFDataCollection::EnsureVTKHDF()
 {

fem/datacollection.hpp

@@ -537,13 +537,6 @@ public:
    /// Any nonzero compression level will enable compression.
    void SetCompressionLevel(int compression_level_);
 
-   /// @brief Enable or disable zlib compression.
-   ///
-   /// If the input is true, use the default zlib compression level (unless the
-   /// compression level has previously been set by calling
-   /// SetCompressionLevel()).
-   void SetCompression(bool compression_) override;
-
    /// @brief Sets whether or not to output the data as high-order elements
    /// (false by default).
    ///
@@ -633,6 +626,12 @@ public:
    ParaViewHDFDataCollection(const std::string& collection_name,
                              Mesh *mesh_ = nullptr);
 
+   /// @brief Enable or disable compression.
+   ///
+   /// The compression level can be set with SetCompressionLevel()). VTKHDF
+   /// compression does not require MFEM to be compiled with zlib support.
+   void SetCompression(bool compression_) override;
+
    /// Save the collection.
    void Save() override;
 

fem/derefmat_op.cpp

@@ -0,0 +1,266 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "derefmat_op.hpp"
+#include "fes_kernels.hpp"
+
+/// \cond DO_NOT_DOCUMENT
+namespace mfem
+{
+
+namespace internal
+{
+template <Ordering::Type Order, bool Atomic>
+static void DerefMultKernelImpl(const DerefineMatrixOp &op, const Vector &x,
+                                Vector &y)
+{
+   DerefineMatrixOpMultFunctor<Order, Atomic> func;
+   func.xptr = x.Read();
+   y.UseDevice();
+   y = 0.;
+   func.yptr = y.ReadWrite();
+   func.bsptr = op.block_storage.Read();
+   func.boptr = op.block_offsets.Read();
+   func.brptr = op.block_row_idcs_offsets.Read();
+   func.bcptr = op.block_col_idcs_offsets.Read();
+   func.rptr = op.row_idcs.Read();
+   func.cptr = op.col_idcs.Read();
+   func.vdims = op.fespace->GetVDim();
+   func.nblocks = op.block_offsets.Size();
+   func.width = op.Width() / func.vdims;
+   func.height = op.Height() / func.vdims;
+   func.Run(op.max_rows);
+}
+
+} // namespace internal
+
+DerefineMatrixOp::DerefineMatrixOp(FiniteElementSpace &fespace_, int old_ndofs,
+                                   const Table *old_elem_dof,
+                                   const Table *old_elem_fos)
+   : Operator(fespace_.GetVSize(), old_ndofs * fespace_.GetVDim()),
+     fespace(&fespace_)
+{
+   static Kernels kernels;
+   constexpr int max_team_size = 256;
+   /// TODO: Implement DofTransformation support
+
+   MFEM_VERIFY(fespace->Nonconforming(),
+               "Not implemented for conforming meshes.");
+   MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
+   MFEM_VERIFY(fespace->GetNDofs() <= old_ndofs,
+               "Previous space is not finer.");
+
+   const CoarseFineTransformations &dtrans =
+      fespace->GetMesh()->ncmesh->GetDerefinementTransforms();
+
+   MFEM_ASSERT(dtrans.embeddings.Size() == old_elem_dof->Size(), "");
+
+   const bool is_dg = fespace->FEColl()->GetContType()
+                      == FiniteElementCollection::DISCONTINUOUS;
+   DenseMatrix localRVO; // for variable-order only
+
+   DenseTensor localR[Geometry::NumGeom];
+   int total_rows = 0;
+   int total_cols = 0;
+   block_offsets.SetSize(dtrans.embeddings.Size());
+   block_offsets.HostWrite();
+   if (fespace->IsVariableOrder())
+   {
+      // TODO: any potential for some compression here?
+      // determine storage size and offsets
+      block_offsets[0] = 0;
+      int total_size = 0;
+      for (int k = 0; k < dtrans.embeddings.Size(); ++k)
+      {
+         const Embedding &emb = dtrans.embeddings[k];
+         const FiniteElement *fe = fespace->GetFE(emb.parent);
+         const int ldof = fe->GetDof();
+         if (k + 1 < dtrans.embeddings.Size())
+         {
+            block_offsets[k + 1] = block_offsets[k] + ldof * ldof;
+         }
+         total_rows += ldof;
+         total_cols += ldof;
+         total_size += ldof * ldof;
+      }
+      block_storage.SetSize(total_size);
+   }
+   else
+   {
+      // compression scheme:
+      // block_offsets is the start of each block, potentially repeated
+      // only need to store localR for used shapes
+      Mesh::GeometryList elem_geoms(*fespace->GetMesh());
+
+      int geom_offsets[Geometry::NumGeom];
+      {
+         int size = 0;
+         for (int i = 0; i < elem_geoms.Size(); ++i)
+         {
+            fespace->GetLocalDerefinementMatrices(elem_geoms[i],
+                                                  localR[elem_geoms[i]]);
+            geom_offsets[elem_geoms[i]] = size;
+            size += localR[elem_geoms[i]].TotalSize();
+         }
+         block_storage.SetSize(size);
+         // copy blocks into block_storage
+         auto bs_ptr = block_storage.HostWrite();
+         for (int i = 0; i < elem_geoms.Size(); ++i)
+         {
+            std::copy(localR[elem_geoms[i]].Data(),
+                      localR[elem_geoms[i]].Data()
+                      + localR[elem_geoms[i]].TotalSize(),
+                      bs_ptr);
+            bs_ptr += localR[elem_geoms[i]].TotalSize();
+         }
+      }
+      for (int k = 0; k < dtrans.embeddings.Size(); ++k)
+      {
+         const Embedding &emb = dtrans.embeddings[k];
+         Geometry::Type geom =
+            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
+
+         auto size = localR[geom].SizeI() * localR[geom].SizeJ();
+         total_rows += localR[geom].SizeI();
+         total_cols += localR[geom].SizeJ();
+         // set block offsets and sizes
+         block_offsets[k] = geom_offsets[geom] + size * emb.matrix;
+      }
+   }
+   row_idcs.SetSize(total_rows);
+   row_idcs.HostWrite();
+   col_idcs.SetSize(total_cols);
+   col_idcs.HostWrite();
+   block_row_idcs_offsets.SetSize(dtrans.embeddings.Size() + 1);
+   block_row_idcs_offsets.HostWrite();
+   block_col_idcs_offsets.SetSize(dtrans.embeddings.Size() + 1);
+   block_col_idcs_offsets.HostWrite();
+   block_row_idcs_offsets[0] = 0;
+   block_col_idcs_offsets[0] = 0;
+
+   // compute index information
+   Array<int> dofs, old_dofs;
+   max_rows = 1;
+
+   {
+      Array<int> mark(fespace->GetNDofs());
+      mark = 0;
+      auto bs_ptr = block_storage.HostWrite();
+      int ridx = 0;
+      int cidx = 0;
+      int num_marked = 0;
+      for (int k = 0; k < dtrans.embeddings.Size(); k++)
+      {
+         const Embedding &emb = dtrans.embeddings[k];
+         Geometry::Type geom =
+            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
+
+         if (fespace->IsVariableOrder())
+         {
+            const FiniteElement *fe = fespace->GetFE(emb.parent);
+            const DenseTensor &pmats = dtrans.point_matrices[geom];
+            const int ldof = fe->GetDof();
+
+            IsoparametricTransformation isotr;
+            isotr.SetIdentityTransformation(geom);
+
+            localRVO.SetSize(ldof, ldof);
+            isotr.SetPointMat(pmats(emb.matrix));
+            // Local restriction is size ldofxldof assuming that the parent
+            // and child are of same polynomial order.
+            fe->GetLocalRestriction(isotr, localRVO);
+            // copy block
+            auto size = localRVO.Height() * localRVO.Width();
+            std::copy(localRVO.Data(), localRVO.Data() + size, bs_ptr);
+            bs_ptr += size;
+         }
+         DenseMatrix &lR =
+            fespace->IsVariableOrder() ? localRVO : localR[geom](emb.matrix);
+         block_row_idcs_offsets[k + 1] =
+            block_row_idcs_offsets[k] + lR.Height();
+         block_col_idcs_offsets[k + 1] = block_col_idcs_offsets[k] + lR.Width();
+         max_rows = std::max(lR.Height(), max_rows);
+         // index information
+         fespace->elem_dof->GetRow(emb.parent, dofs);
+         old_elem_dof->GetRow(k, old_dofs);
+         MFEM_VERIFY(old_dofs.Size() == dofs.Size(),
+                     "Parent and child must have same #dofs.");
+         for (int i = 0; i < lR.Height(); ++i, ++ridx)
+         {
+            if (!std::isfinite(lR(i, 0)))
+            {
+               row_idcs[ridx] = INT_MAX;
+               continue;
+            }
+            int r = dofs[i];
+            int m = (r >= 0) ? r : (-1 - r);
+            if (is_dg || !mark[m])
+            {
+               row_idcs[ridx] = r;
+               mark[m] = 1;
+               ++num_marked;
+            }
+            else
+            {
+               row_idcs[ridx] = INT_MAX;
+            }
+         }
+         for (int i = 0; i < lR.Width(); ++i, ++cidx)
+         {
+            col_idcs[cidx] = old_dofs[i];
+         }
+      }
+      if (!is_dg && !fespace->IsVariableOrder())
+      {
+         MFEM_VERIFY(num_marked * fespace->GetVDim() == Height(),
+                     "internal error: not all rows were set.");
+      }
+   }
+   // if not using GPU, set max_rows/max_cols to zero
+   if (Device::Allows(Backend::DEVICE_MASK))
+   {
+      max_rows = std::min(max_rows, max_team_size);
+   }
+   else
+   {
+      max_rows = 1;
+   }
+}
+
+void DerefineMatrixOp::Mult(const Vector &x, Vector &y) const
+{
+   const bool is_dg = fespace->FEColl()->GetContType()
+                      == FiniteElementCollection::DISCONTINUOUS;
+   // DG needs atomic summation
+   MultKernel::Run(fespace->GetOrdering(), is_dg, *this, x, y);
+}
+
+DerefineMatrixOp::Kernels::Kernels()
+{
+   MultKernel::Specialization<Ordering::byNODES, false>::Add();
+   MultKernel::Specialization<Ordering::byVDIM, false>::Add();
+   MultKernel::Specialization<Ordering::byNODES, true>::Add();
+   MultKernel::Specialization<Ordering::byVDIM, true>::Add();
+}
+
+template <Ordering::Type Order, bool Atomic>
+DerefineMatrixOp::MultKernelType DerefineMatrixOp::MultKernel::Kernel()
+{
+   return internal::DerefMultKernelImpl<Order, Atomic>;
+}
+
+DerefineMatrixOp::MultKernelType
+DerefineMatrixOp::MultKernel::Fallback(Ordering::Type, bool)
+{
+   MFEM_ABORT("invalid MultKernel parameters");
+}
+} // namespace mfem
+/// \endcond DO_NOT_DOCUMENT

fem/derefmat_op.hpp

@@ -0,0 +1,65 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DEREFMAT_OP
+#define MFEM_DEREFMAT_OP
+
+#include "fespace.hpp"
+
+#include "kernel_dispatch.hpp"
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+
+struct DerefineMatrixOp : public Operator
+{
+   FiniteElementSpace *fespace;
+   /// offsets into block_storage
+   Array<int> block_offsets;
+   /// offsets into row_idcs
+   Array<int> block_row_idcs_offsets;
+   /// offsets into col_idcs
+   Array<int> block_col_idcs_offsets;
+   /// mapping for row dofs, INT_MAX indicates the block row should be ignored.
+   /// negative means the row data should be negated.
+   Array<int> row_idcs;
+   /// mapping for col dofs, negative means the col data should be negated.
+   Array<int> col_idcs;
+   /// dense block matrices which can be reused to construct the full matrix
+   /// operation. These are stored contiguously and blocks have no restrictions
+   /// on shape (can be rectangle and differ from block to block).
+   Vector block_storage;
+   /// maximum height of any block in block_storage for GPU
+   /// parallelization, or 1 for CPU runs.
+   int max_rows;
+
+   using MultKernelType = void (*)(const DerefineMatrixOp &, const Vector &,
+                                   Vector &);
+   /// template args: ordering, atomic
+   MFEM_REGISTER_KERNELS(MultKernel, MultKernelType, (Ordering::Type, bool));
+
+   struct Kernels
+   {
+      Kernels();
+   };
+
+   void Mult(const Vector &x, Vector &y) const;
+
+   DerefineMatrixOp(FiniteElementSpace &fespace_, int old_ndofs,
+                    const Table *old_elem_dof, const Table *old_elem_fos);
+};
+
+/// \endcond DO_NOT_DOCUMENT
+
+} // namespace mfem
+#endif

fem/dfem/doperator.hpp

@@ -231,21 +231,53 @@ public:
       const std::vector<FieldDescriptor> &parameters,
       const ParMesh &mesh);
 
+   /// MultLevel enum to indicate if the T->L Operators are used in the
+   /// Mult method.
+   enum MultLevel
+   {
+      TVECTOR,
+      LVECTOR
+   };
+
+   /// @brief Set the MultLevel mode for the DifferentiableOperator.
+   /// The default is TVECTOR, which means that the Operator will use
+   /// T->L before Mult and L->T Operators after.
+   void SetMultLevel(MultLevel level)
+   {
+      mult_level = level;
+   }
+
    /// @brief Compute the action of the operator on a given vector.
    ///
-   /// @param solutions_t The solution vector in which to compute the action.
-   /// This has to be a T-dof vector.
-   /// @param result_t Result vector of the action of the operator on
-   /// solutions_t. The result is a T-dof vector.
-   void Mult(const Vector &solutions_t, Vector &result_t) const override
+   /// @param solutions_in The solution vector in which to compute the action.
+   /// This has to be a T-dof vector if MultLevel is set to TVECTOR, or L-dof
+   /// Vector if MultLevel is set to LVECTOR.
+   /// @param result_in Result vector of the action of the operator on
+   /// solutions. The result is a T-dof vector or L-dof vector depending on
+   /// the MultLevel.
+   void Mult(const Vector &solutions_in, Vector &result_in) const override
    {
       MFEM_ASSERT(!action_callbacks.empty(), "no integrators have been set");
-      prolongation(solutions, solutions_t, solutions_l);
-      for (auto &action : action_callbacks)
+
+      if (mult_level == MultLevel::LVECTOR)
       {
-         action(solutions_l, parameters_l, residual_l);
+         get_lvectors(solutions, solutions_in, solutions_l);
+         result_in = 0.0;
+         for (auto &action : action_callbacks)
+         {
+            action(solutions_l, parameters_l, result_in);
+         }
+      }
+      else
+      {
+         prolongation(solutions, solutions_in, solutions_l);
+         residual_l = 0.0;
+         for (auto &action : action_callbacks)
+         {
+            action(solutions_l, parameters_l, residual_l);
+         }
+         prolongation_transpose(residual_l, result_in);
       }
-      prolongation_transpose(residual_l, result_t);
    }
 
    /// @brief Add a domain integrator to the operator.
@@ -344,6 +376,8 @@ public:
 private:
    const ParMesh &mesh;
 
+   MultLevel mult_level = TVECTOR;
+
    std::vector<action_t> action_callbacks;
    std::map<size_t,
        std::vector<derivative_action_t>> derivative_action_callbacks;
@@ -353,7 +387,6 @@ private:
        std::vector<assemble_derivative_hypreparmatrix_callback_t>>
        assemble_derivative_hypreparmatrix_callbacks;
 
-
    std::vector<FieldDescriptor> solutions;
    std::vector<FieldDescriptor> parameters;
    // solutions and parameters

fem/dfem/parameterspace.hpp

@@ -101,10 +101,11 @@ public:
       // Setup DofToQuad information
       dtq.nqpt = (int)floor(std::pow(ir.GetNPoints(), 1.0 / mesh.Dimension()) + 0.5);
       dtq.ndof = dtq.nqpt;
+      dtq.mode = used_in_tensor_product ? DofToQuad::TENSOR : DofToQuad::FULL;
 
       // Calculate sizes
       const int num_qp = used_in_tensor_product ?
-                         std::pow(dtq.nqpt, mesh.Dimension()) :
+                         static_cast<int>(std::pow(dtq.nqpt, mesh.Dimension())) :
                          ir.GetNPoints();
 
       tsize = vdim * num_qp * mesh.GetNE();

fem/dfem/util.hpp

@@ -327,8 +327,8 @@ void print_mpi_sync(const std::string& msg)
    // First gather string lengths
    size_t msg_len = msg.length();
    std::vector<size_t> lengths(nranks);
-   MPI_Gather(&msg_len, 1, MPI_INT,
-              lengths.data(), 1, MPI_INT,
+   MPI_Gather(&msg_len, 1, MPITypeMap<size_t>::mpi_type,
+              lengths.data(), 1, MPITypeMap<size_t>::mpi_type,
               0, MPI_COMM_WORLD);
 
    if (myrank == 0)
@@ -568,7 +568,7 @@ struct ThreadBlocks
    int z = 1;
 };
 
-#if (defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+#if defined(MFEM_USE_CUDA_OR_HIP)
 template <typename func_t>
 __global__ void forall_kernel_shmem(func_t f, int n)
 {
@@ -591,7 +591,7 @@ void forall(func_t f,
    if (Device::Allows(Backend::CUDA_MASK) ||
        Device::Allows(Backend::HIP_MASK))
    {
-#if (defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+#if defined(MFEM_USE_CUDA_OR_HIP)
       // int gridsize = (N + Z - 1) / Z;
       int num_bytes = num_shmem * sizeof(decltype(shmem));
       dim3 block_size(blocks.x, blocks.y, blocks.z);
@@ -987,7 +987,7 @@ get_restriction_transpose(
    {
       auto RT = [=](const Vector &v_e, Vector &v_l)
       {
-         v_l = v_e;
+         v_l += v_e;
       };
       return std::make_tuple(RT, 1);
    }
@@ -996,7 +996,7 @@ get_restriction_transpose(
       const Operator *R = get_restriction<entity_t>(f, o);
       std::function<void(const Vector&, Vector&)> RT = [=](const Vector &x, Vector &y)
       {
-         R->MultTranspose(x, y);
+         R->AddMultTranspose(x, y);
       };
       return std::make_tuple(RT, R->Height());
    }
@@ -1076,6 +1076,24 @@ void prolongation(const std::vector<FieldDescriptor> fields,
    }
 }
 
+inline
+void get_lvectors(const std::vector<FieldDescriptor> fields,
+                  const Vector &x,
+                  std::vector<Vector> &fields_l)
+{
+   int data_offset = 0;
+   for (std::size_t i = 0; i < fields.size(); i++)
+   {
+      const int sz = GetVSize(fields[i]);
+      fields_l[i].SetSize(sz);
+
+      const Vector x_i(const_cast<Vector&>(x), data_offset, sz);
+      fields_l[i] = x_i;
+
+      data_offset += sz;
+   }
+}
+
 /// @brief Get a transpose prolongation callback for a field descriptor.
 ///
 /// In the special case of a one field operator, the transpose prolongation
@@ -1702,12 +1720,13 @@ std::array<DofToQuadMap, N> load_dtq_mem(
    std::array<DofToQuadMap, N> f;
    for (std::size_t i = 0; i < N; i++)
    {
-      const auto [nqp_b, dim_b, ndof_b] = dtq[i].B.GetShape();
-      const auto B = Reshape(&dtq[i].B[0], nqp_b, dim_b, ndof_b);
-      auto mem_Bi = Reshape(reinterpret_cast<real_t *>(mem) + offset, nqp_b, dim_b,
-                            ndof_b);
       if (dtq[i].which_input != -1)
       {
+         const auto [nqp_b, dim_b, ndof_b] = dtq[i].B.GetShape();
+         const auto B = Reshape(&dtq[i].B[0], nqp_b, dim_b, ndof_b);
+         auto mem_Bi = Reshape(reinterpret_cast<real_t *>(mem) + offset, nqp_b, dim_b,
+                               ndof_b);
+
          MFEM_FOREACH_THREAD(q, x, nqp_b)
          {
             MFEM_FOREACH_THREAD(d, y, ndof_b)
@@ -2158,7 +2177,7 @@ template <
    std::size_t... Is>
 std::array<DofToQuadMap, N> create_dtq_maps_impl(
    field_operator_ts &fops,
-   std::vector<const DofToQuad*> dtqs,
+   std::vector<const DofToQuad*> &dtqs,
    const std::array<int, N> &field_map,
    std::index_sequence<Is...>)
 {
@@ -2243,7 +2262,7 @@ template <
    std::size_t num_fields>
 std::array<DofToQuadMap, num_fields> create_dtq_maps(
    field_operator_ts &fops,
-   std::vector<const DofToQuad*> dtqmaps,
+   std::vector<const DofToQuad*> &dtqmaps,
    const std::array<int, num_fields> &to_field_map)
 {
    return create_dtq_maps_impl<entity_t>(

fem/fes_kernels.hpp

@@ -0,0 +1,249 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FES_KERNELS_HPP
+#define MFEM_FES_KERNELS_HPP
+
+#include "../general/forall.hpp"
+
+#include <climits>
+
+namespace mfem
+{
+/// \cond DO_NOT_DOCUMENT
+namespace internal
+{
+
+///
+/// Implements matrix-vector multiply $y = A x$ for a sparse matrix composed of
+/// a sum of smaller dense blocks. There is additional permutation/sign
+/// information associated with each block. The base class only implements
+/// helper routines such as computing block widths, index into x, index into y,
+/// and column in A given sub-block information.
+/// @sa DerefineMatrixOpMultFunctor
+///
+/// @tparam Order vdim ordering for x and y. Note that for Diag = false this is
+/// ignored for x as x has a special interleaved order.
+/// @tparam Base used for the curious recurring template pattern (CRTP) so the
+/// base class can access child class fields without virtual functions
+/// @tparam Diag true if this corresponds to the diagonal block (coarse element
+/// and fine element are on our rank), false otherwise (coarse element is on our
+/// rank, fine element is on a different rank).
+///
+template <Ordering::Type Order, class Base, bool Diag = true>
+struct DerefineMatrixOpFunctorBase;
+
+template <class Base>
+struct DerefineMatrixOpFunctorBase<Ordering::byNODES, Base, true>
+{
+   /// block column indices offsets
+   const int *bcptr;
+   /// column indices
+   const int *cptr;
+
+   int MFEM_HOST_DEVICE BlockWidth(int k) const
+   {
+      return bcptr[k + 1] - bcptr[k];
+   }
+
+   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
+   {
+      col = cptr[bcptr[k] + j];
+      if (col < 0)
+      {
+         col = -1 - col;
+         sign = -sign;
+      }
+   }
+
+   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int) const
+   {
+      return col + vdim * static_cast<const Base *>(this)->width;
+   }
+   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
+   {
+      return row + vdim * static_cast<const Base *>(this)->height;
+   }
+};
+
+template <class Base>
+struct DerefineMatrixOpFunctorBase<Ordering::byVDIM, Base, true>
+{
+   /// block column indices offsets
+   const int *bcptr;
+   /// column indices
+   const int *cptr;
+
+   int MFEM_HOST_DEVICE BlockWidth(int k) const
+   {
+      return bcptr[k + 1] - bcptr[k];
+   }
+
+   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
+   {
+      col = cptr[bcptr[k] + j];
+      if (col < 0)
+      {
+         col = -1 - col;
+         sign = -sign;
+      }
+   }
+
+   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int) const
+   {
+      return vdim + col * static_cast<const Base *>(this)->vdims;
+   }
+   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
+   {
+      return vdim + row * static_cast<const Base *>(this)->vdims;
+   }
+};
+
+template <class Base>
+struct DerefineMatrixOpFunctorBase<Ordering::byNODES, Base, false>
+{
+   /// receive segment offsets
+   const int *segptr;
+   /// receive segment index
+   const int *rsptr;
+   /// off-diagonal block column offsets
+   const int *coptr;
+   /// off-diagonal block widths
+   const int *bwptr;
+
+   int MFEM_HOST_DEVICE BlockWidth(int k) const { return bwptr[k]; }
+
+   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
+   {
+      col = coptr[k] + j;
+   }
+
+   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int k) const
+   {
+      int tmp = rsptr[k];
+      int segwidth = segptr[tmp + 1] - segptr[tmp];
+      return segptr[tmp] * static_cast<const Base *>(this)->vdims + col +
+             vdim * segwidth;
+   }
+   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
+   {
+      return row + vdim * static_cast<const Base *>(this)->height;
+   }
+};
+
+template <class Base>
+struct DerefineMatrixOpFunctorBase<Ordering::byVDIM, Base, false>
+{
+   /// receive segment offsets
+   const int *segptr;
+   /// receive segment index
+   const int *rsptr;
+   /// off-diagonal block column offsets
+   const int *coptr;
+   /// off-diagonal block widths
+   const int *bwptr;
+
+   int MFEM_HOST_DEVICE BlockWidth(int k) const { return bwptr[k]; }
+
+   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
+   {
+      col = coptr[k] + j;
+   }
+
+   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int k) const
+   {
+      int tmp = rsptr[k];
+      int segwidth = segptr[tmp + 1] - segptr[tmp];
+      return segptr[tmp] * static_cast<const Base *>(this)->vdims + col +
+             vdim * segwidth;
+   }
+   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
+   {
+      return vdim + row * static_cast<const Base *>(this)->vdims;
+   }
+};
+
+/// internally used to implement the derefinement operator Mult diagonal
+/// block
+template <Ordering::Type Order, bool Atomic, bool Diag = true>
+struct DerefineMatrixOpMultFunctor
+   : public DerefineMatrixOpFunctorBase<
+     Order, DerefineMatrixOpMultFunctor<Order, Atomic, Diag>, Diag>
+{
+   const real_t *xptr;
+   real_t *yptr;
+   /// block storage
+   const real_t *bsptr;
+   /// block offsets
+   const int *boptr;
+   /// block row index offsets
+   const int *brptr;
+   /// row indices
+   const int *rptr;
+
+   // number of blocks
+   int nblocks;
+   // number of components
+   int vdims;
+   /// overall operator height (for vdim = 1)
+   int height;
+   /// overall operator width (for vdim = 1)
+   int width;
+   void MFEM_HOST_DEVICE operator()(int kidx) const
+   {
+      int k = kidx % nblocks;
+      int vdim = kidx / nblocks;
+
+      int block_height = brptr[k + 1] - brptr[k];
+      int block_width = this->BlockWidth(k);
+      MFEM_FOREACH_THREAD(i, x, block_height)
+      {
+         int row = rptr[brptr[k] + i];
+         int rsign = 1;
+         if (row < 0)
+         {
+            row = -1 - row;
+            rsign = -1;
+         }
+         if (row < INT_MAX)
+         {
+            // row not marked as unused
+            real_t sum = 0;
+            for (int j = 0; j < block_width; ++j)
+            {
+               int col, sign = rsign;
+               this->Col(j, k, col, sign);
+               sum += sign * bsptr[boptr[k] + i + j * block_height] *
+                      xptr[this->IndexX(col, vdim, k)];
+            }
+#if defined(__CUDA_ARCH__) or defined(__HIP_DEVICE_COMPILE__)
+            if (Atomic)
+            {
+               atomicAdd(yptr + this->IndexY(row, vdim), sum);
+            }
+            else
+#endif
+            {
+               yptr[this->IndexY(row, vdim)] += sum;
+            }
+         }
+      }
+   }
+
+   /// N is the max block row size (doesn't have to be a power of 2)
+   void Run(int N) const { forall_2D(nblocks * vdims, N, 1, *this); }
+};
+
+} // namespace internal
+/// \endcond DO_NOT_DOCUMENT
+} // namespace mfem
+
+#endif

fem/fespace.cpp

@@ -17,6 +17,9 @@
 #include "fem.hpp"
 #include "ceed/interface/util.hpp"
 
+#include "derefmat_op.hpp"
+
+#include <algorithm>
 #include <cmath>
 #include <cstdarg>
 
@@ -24,9 +27,9 @@ using namespace std;
 
 namespace mfem
 {
-
-template <> void Ordering::
-DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim, Array<int> &dofs)
+template <>
+void Ordering::DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim,
+                                              Array<int> &dofs)
 {
    // static method
    int size = dofs.Size();
@@ -40,8 +43,9 @@ DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim, Array<int> &dofs)
    }
 }
 
-template <> void Ordering::
-DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)
+template <>
+void Ordering::DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim,
+                                             Array<int> &dofs)
 {
    // static method
    int size = dofs.Size();
@@ -55,7 +59,6 @@ DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)
    }
 }
 
-
 FiniteElementSpace::FiniteElementSpace()
    : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
      ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
@@ -4244,7 +4247,11 @@ void FiniteElementSpace::Update(bool want_transform)
          case Mesh::DEREFINE:
          {
             BuildConformingInterpolation();
+#if 0
             Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
+#else
+            Th.Reset(new DerefineMatrixOp(*this, old_ndofs, old_elem_dof, old_elem_fos));
+#endif
             if (IsVariableOrder())
             {
                if (cP && cR_hp)

fem/fespace.hpp

@@ -113,7 +113,7 @@ class QuadratureSpace;
 class QuadratureInterpolator;
 class FaceQuadratureInterpolator;
 class PRefinementTransferOperator;
-
+struct DerefineMatrixOp;
 
 /** @brief Class FiniteElementSpace - responsible for providing FEM view of the
     mesh, mainly managing the set of degrees of freedom.
@@ -246,6 +246,7 @@ class FiniteElementSpace
    friend class PRefinementTransferOperator;
    friend void Mesh::Swap(Mesh &, bool);
    friend class LORBase;
+   friend struct DerefineMatrixOp;
 
 protected:
    /// The mesh that FE space lives on (not owned).

fem/gridfunc.cpp

@@ -4334,7 +4334,7 @@ real_t LSZZErrorEstimator(BilinearFormIntegrator &blfi,  // input
          u.GetSubVector(udofs, ul);
          utrans.InvTransformPrimal(ul);
          Transf = ufes->GetElementTransformation(ielem);
-         FiniteElement *dummy = nullptr;
+         const auto *dummy = ufes->GetFE(ielem);
          blfi.ComputeElementFlux(*ufes->GetFE(ielem), *Transf, ul,
                                  *dummy, fl, with_coeff, ir);
 

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -1009,6 +1009,7 @@ inline void SmemPADiffusionApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_VERIFY(D1D <= Q1D, "THREAD_DIRECT requires D1D <= Q1D");
    mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -1038,11 +1039,11 @@ inline void SmemPADiffusionApply3D(const int NE,
       real_t (*QDD0)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+0);
       real_t (*QDD1)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+1);
       real_t (*QDD2)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
@@ -1050,9 +1051,9 @@ inline void SmemPADiffusionApply3D(const int NE,
       }
       if (MFEM_THREAD_ID(z) == 0)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                B[qx][dy] = b(qx,dy);
                G[qx][dy] = g(qx,dy);
@@ -1060,11 +1061,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                real_t u = 0.0, v = 0.0;
                MFEM_UNROLL(MD1)
@@ -1080,11 +1081,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MD1)
@@ -1101,11 +1102,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MD1)
@@ -1136,9 +1137,9 @@ inline void SmemPADiffusionApply3D(const int NE,
       MFEM_SYNC_THREAD;
       if (MFEM_THREAD_ID(z) == 0)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_FOREACH_THREAD_DIRECT(qx,x,Q1D)
             {
                Bt[dy][qx] = b(qx,dy);
                Gt[dy][qx] = g(qx,dy);
@@ -1146,11 +1147,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD_DIRECT(qy,y,Q1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MQ1)
@@ -1167,11 +1168,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD_DIRECT(qz,z,Q1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(Q1D)
@@ -1188,11 +1189,11 @@ inline void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD_DIRECT(dz,z,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD_DIRECT(dy,y,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_FOREACH_THREAD_DIRECT(dx,x,D1D)
             {
                real_t u = 0.0, v = 0.0, w = 0.0;
                MFEM_UNROLL(MQ1)

fem/integ/bilininteg_mass_pa.cpp

@@ -59,26 +59,23 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    QuadratureSpace qs(*mesh, *ir);
    CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
-   const int NE = ne;
-   const int Q1D = quad1D;
-   const int NQ = pow(Q1D, dim);
-   const bool const_c = coeff.Size() == 1;
-   const bool by_val = map_type == FiniteElement::VALUE;
-   const auto W = Reshape(ir->GetWeights().Read(), NQ);
-   const auto J = Reshape(geom->detJ.Read(), NQ, NE);
-   const auto C = const_c ? Reshape(coeff.Read(), 1, 1) :
-                  Reshape(coeff.Read(), NQ,NE);
-   auto v = Reshape(pa_data.Write(), NQ, NE);
-   mfem::forall_2D(NE, NQ, 1, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_FOREACH_THREAD(i, x, NQ)
+      const int NE = ne;
+      const int NQ = nq;
+      const bool const_c = coeff.Size() == 1;
+      const bool by_val = map_type == FiniteElement::VALUE;
+      const auto W = Reshape(ir->GetWeights().Read(), NQ);
+      const auto J = Reshape(geom->detJ.Read(), NQ, NE);
+      const auto C =
+         const_c ? Reshape(coeff.Read(), 1, 1) : Reshape(coeff.Read(), NQ, NE);
+      auto v = Reshape(pa_data.Write(), NQ, NE);
+      mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e)
       {
-         const real_t detJ = J(i,e);
-         const real_t coeff = const_c ? C(0,0) : C(i,e);
-         v(i,e) =  W(i) * coeff * (by_val ? detJ : 1.0/detJ);
-      }
-   });
+         const real_t detJ = J(q, e);
+         const real_t coeff = const_c ? C(0, 0) : C(q, e);
+         v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ);
+      });
+   }
 }
 
 void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
@@ -109,50 +106,22 @@ void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
    CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
 
    const int NE = ne;
-   const int Q1D = quad1D;
+   const int NQ = nq;
    const bool const_c = coeff.Size() == 1;
    const bool by_val = map_type == FiniteElement::VALUE;
-   if (dim==1)
    {
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D);
-      const auto J = Reshape(face_geom->detJ.Read(), Q1D, NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1, 1) :
-                     Reshape(coeff.Read(), Q1D, NE);
-      auto v = Reshape(pa_data.Write(), Q1D, NE);
-      mfem::forall_2D(NE, Q1D, 1, [=] MFEM_HOST_DEVICE (int e)
+      const auto W = Reshape(ir->GetWeights().Read(), NQ);
+      const auto J = Reshape(face_geom->detJ.Read(), NQ, NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1, 1)
+                     : Reshape(coeff.Read(), NQ, NE);
+      auto v = Reshape(pa_data.Write(), NQ, NE);
+      mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e)
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const real_t detJ = J(qx,e);
-            const real_t coeff = const_c ? C(0,0) : C(qx,e);
-            v(qx,e) =  W(qx) * coeff * (by_val ? detJ : 1.0/detJ);
-         }
+         const real_t detJ = J(q, e);
+         const real_t coeff = const_c ? C(0, 0) : C(q, e);
+         v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ);
       });
    }
-   else if (dim==2)
-   {
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
-      const auto J = Reshape(face_geom->detJ.Read(), Q1D,Q1D,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
-      mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               const real_t detJ = J(qx,qy,e);
-               const real_t coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
-            }
-         }
-      });
-   }
-   else
-   {
-      MFEM_ABORT("Not supported.");
-   }
 }
 
 void MassIntegrator::AssembleDiagonalPA(Vector &diag)

fem/intrules_cut.hpp

@@ -346,13 +346,13 @@ private:
       template<typename T>
       T operator() (const blitz::TinyVector<T,3>& x) const
       {
-         int el_order=el->GetOrder();
-         T u1[el_order+1];
-         T u2[el_order+1];
-         T u3[el_order+1];
-         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
-         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);
-         TmplPoly_1D::CalcBernstein(el_order, x[2], u3);
+         const int el_order = el->GetOrder();
+         std::vector<T> u1(el_order+1);
+         std::vector<T> u2(el_order+1);
+         std::vector<T> u3(el_order+1);
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1.data());
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2.data());
+         TmplPoly_1D::CalcBernstein(el_order, x[2], u3.data());
 
          const Array<int>& dof_map=el->GetDofMap();
 
@@ -370,17 +370,17 @@ private:
       template<typename T>
       blitz::TinyVector<T,3> grad(const blitz::TinyVector<T,3>& x) const
       {
-         int el_order=el->GetOrder();
-         T u1[el_order+1];
-         T u2[el_order+1];
-         T u3[el_order+1];
-         T d1[el_order+1];
-         T d2[el_order+1];
-         T d3[el_order+1];
+         const int el_order = el->GetOrder();
+         std::vector<T> u1(el_order+1);
+         std::vector<T> u2(el_order+1);
+         std::vector<T> u3(el_order+1);
+         std::vector<T> d1(el_order+1);
+         std::vector<T> d2(el_order+1);
+         std::vector<T> d3(el_order+1);
 
-         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
-         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);
-         TmplPoly_1D::CalcBernstein(el_order,x[2], u3, d3);
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1.data(), d1.data());
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2.data(), d2.data());
+         TmplPoly_1D::CalcBernstein(el_order,x[2], u3.data(), d3.data());
 
          blitz::TinyVector<T,3> res(T(0.0),T(0.0),T(0.0));
 
@@ -415,11 +415,11 @@ private:
       template<typename T>
       T operator() (const blitz::TinyVector<T,2>& x) const
       {
-         int el_order=el->GetOrder();
-         T u1[el_order+1];
-         T u2[el_order+1];
-         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
-         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);
+         const int el_order = el->GetOrder();
+         std::vector<T> u1(el_order+1);
+         std::vector<T> u2(el_order+1);
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1.data());
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2.data());
 
          const Array<int>& dof_map=el->GetDofMap();
 
@@ -437,14 +437,14 @@ private:
       template<typename T>
       blitz::TinyVector<T,2> grad(const blitz::TinyVector<T,2>& x) const
       {
-         int el_order=el->GetOrder();
-         T u1[el_order+1];
-         T u2[el_order+1];
-         T d1[el_order+1];
-         T d2[el_order+1];
+         const int el_order = el->GetOrder();
+         std::vector<T> u1(el_order+1);
+         std::vector<T> u2(el_order+1);
+         std::vector<T> d1(el_order+1);
+         std::vector<T> d2(el_order+1);
 
-         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
-         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1.data(), d1.data());
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2.data(), d2.data());
 
          blitz::TinyVector<T,2> res(T(0.0),T(0.0));
 

fem/linearform_ext.cpp

@@ -51,7 +51,7 @@ void LinearFormExtension::Assemble()
       {
          // scan the attributes to set the markers to 0 or 1
          const int NE = fes.GetNE();
-         const auto attr = attributes.Read();
+         const auto attr = attributes->Read();
          const auto dimk = domain_integs_marker_k->Read();
          auto markers_w = markers.Write();
          mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
@@ -93,13 +93,14 @@ void LinearFormExtension::Assemble()
       else
       {
          // scan the attributes to set the markers to 0 or 1
-         const int NBE = bdr_attributes.Size();
-         const auto attr = bdr_attributes.Read();
+         const int NBE = bdr_face_attributes->Size();
+         const auto attr = bdr_face_attributes->Read();
          const auto attr_markers = boundary_integs_marker_k->Read();
          auto markers_w = bdr_markers.Write();
-         mfem::forall(NBE, [=] MFEM_HOST_DEVICE (int e)
+         mfem::forall(NBE, [=] MFEM_HOST_DEVICE(int e)
          {
-            markers_w[e] = attr_markers[attr[e]-1] == 1;
+            markers_w[e] =
+               attr[e] > 0 ? (attr_markers[attr[e] - 1] == 1) : false;
          });
       }
 
@@ -125,8 +126,7 @@ void LinearFormExtension::Update()
       //markers.UseDevice(true);
 
       // Gather the attributes on the host from all the elements
-      attributes.SetSize(NE);
-      for (int i = 0; i < NE; ++i) { attributes[i] = mesh.GetAttribute(i); }
+      attributes = &mesh.GetElementAttributes();
 
       elem_restrict_lex = fes.GetElementRestriction(ordering);
       MFEM_VERIFY(elem_restrict_lex, "Element restriction not available");
@@ -136,34 +136,12 @@ void LinearFormExtension::Update()
 
    if (lf->boundary_integs.Size() > 0)
    {
-      const int nf_bdr = fes.GetNFbyType(FaceType::Boundary);
+      bdr_face_attributes = &mesh.GetBdrFaceAttributes();
+
+      const int nf_bdr = bdr_face_attributes->Size();
       bdr_markers.SetSize(nf_bdr);
       // bdr_markers.UseDevice(true);
 
-      // The face restriction will give us "face E-vectors" on the boundary that
-      // are numbered in the order of the faces of mesh. This numbering will be
-      // different than the numbering of the boundary elements. We compute
-      // mappings so that the array `bdr_attributes[i]` gives the boundary
-      // attribute of the `i`th boundary face in the mesh face order.
-      std::unordered_map<int,int> f_to_be;
-      for (int i = 0; i < mesh.GetNBE(); ++i)
-      {
-         const int f = mesh.GetBdrElementFaceIndex(i);
-         f_to_be[f] = i;
-      }
-      MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
-      bdr_attributes.SetSize(nf_bdr);
-      int f_ind = 0;
-      for (int f = 0; f < mesh.GetNumFaces(); ++f)
-      {
-         if (f_to_be.find(f) != f_to_be.end())
-         {
-            const int be = f_to_be[f];
-            bdr_attributes[f_ind] = mesh.GetBdrAttribute(be);
-            ++f_ind;
-         }
-      }
-
       bdr_restrict_lex =
          dynamic_cast<const FaceRestriction*>(
             fes.GetFaceRestriction(ordering, FaceType::Boundary,

fem/linearform_ext.hpp

@@ -25,7 +25,8 @@ class LinearForm;
 class LinearFormExtension
 {
    /// Attributes of all mesh elements.
-   Array<int> attributes, bdr_attributes;
+   const Array<int> *attributes; // Not owned
+   const Array<int> *bdr_face_attributes; // Not owned
 
    /// Temporary markers for device kernels.
    Array<int> markers, bdr_markers;

fem/lininteg.hpp

@@ -673,7 +673,7 @@ public:
       int myid;
       MPI_Comm_rank(comm, &myid);
 
-      int seed = (seed_ > 0) ? seed_ + myid : (int)time(0) + myid;
+      int seed = (seed_ > 0) ? seed_ + myid : time(nullptr) + myid;
       SetSeed(seed);
    }
 #else

fem/pderefmat_op.cpp

@@ -0,0 +1,591 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "pderefmat_op.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "fes_kernels.hpp"
+/// \cond DO_NOT_DOCUMENT
+namespace mfem
+{
+namespace internal
+{
+template <Ordering::Type Order, bool Atomic>
+static void ParDerefMultKernelImpl(const ParDerefineMatrixOp &op,
+                                   const Vector &x, Vector &y)
+{
+   // pack sends
+   if (op.xghost_send.Size())
+   {
+      auto src = x.Read();
+      auto idcs = op.send_permutations.Read();
+      auto dst = Device::GetGPUAwareMPI() ? op.xghost_send.Write()
+                 : op.xghost_send.HostWrite();
+      auto vdims = op.fespace->GetVDim();
+      auto sptr = op.send_segment_idcs.Read();
+      auto lptr = op.send_segments.Read();
+      auto old_ndofs = x.Size() / vdims;
+
+      forall(op.send_permutations.Size(), [=] MFEM_HOST_DEVICE(int i)
+      {
+         int seg = sptr[i];
+         int width = lptr[seg + 1] - lptr[seg];
+         auto tdst = dst + i + lptr[seg] * vdims;
+         int sign = 1;
+         int col = idcs[i];
+         if (col < 0)
+         {
+            sign = -1;
+            col = -1 - col;
+         }
+         for (int vdim = 0; vdim < vdims; ++vdim)
+         {
+            tdst[vdim * width] =
+               sign
+               * src[Order == Ordering::byNODES ? (col + vdim * old_ndofs)
+                           : (col * vdims + vdim)];
+         }
+      });
+      // TODO: is this needed so we can send the packed data correctly?
+      // unclear for GPU-aware MPI, definitely required otherwise
+      MFEM_DEVICE_SYNC;
+   }
+   // initialize off-diagonal receive and send
+   op.requests.clear();
+   if (op.xghost_recv.Size())
+   {
+      auto vdims = op.fespace->GetVDim();
+      auto rcv = Device::GetGPUAwareMPI() ? op.xghost_recv.Write()
+                 : op.xghost_recv.HostWrite();
+      for (int i = 0; i < op.recv_ranks.Size(); ++i)
+      {
+         op.requests.emplace_back();
+         MPI_Irecv(rcv + op.recv_segments[i] * vdims,
+                   (op.recv_segments[i + 1] - op.recv_segments[i]) * vdims,
+                   MPITypeMap<real_t>::mpi_type, op.recv_ranks[i],
+                   MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA,
+                   op.fespace->GetComm(), &op.requests.back());
+      }
+   }
+   if (op.xghost_send.Size())
+   {
+      auto vdims = op.fespace->GetVDim();
+      // only is a GPU mem ptr if GPU-aware MPI is enabled
+      auto dst = Device::GetGPUAwareMPI() ? op.xghost_send.Write()
+                 : op.xghost_send.HostWrite();
+      for (int i = 0; i < op.send_ranks.Size(); ++i)
+      {
+         op.requests.emplace_back();
+         MPI_Isend(dst + op.send_segments[i] * vdims,
+                   (op.send_segments[i + 1] - op.send_segments[i]) * vdims,
+                   MPITypeMap<real_t>::mpi_type, op.send_ranks[i],
+                   MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA,
+                   op.fespace->GetComm(), &op.requests.back());
+      }
+   }
+   {
+      // diagonal
+      DerefineMatrixOpMultFunctor<Order, Atomic, true> func;
+      func.xptr = x.Read();
+      y.UseDevice();
+      y = 0.;
+      func.yptr = y.ReadWrite();
+      func.bsptr = op.block_storage.Read();
+      func.boptr = op.block_offsets.Read();
+      func.brptr = op.block_row_idcs_offsets.Read();
+      func.bcptr = op.block_col_idcs_offsets.Read();
+      func.rptr = op.row_idcs.Read();
+      func.cptr = op.col_idcs.Read();
+      func.vdims = op.fespace->GetVDim();
+      func.nblocks = op.block_offsets.Size();
+      func.width = op.Width() / func.vdims;
+      func.height = op.Height() / func.vdims;
+      func.Run(op.max_rows);
+   }
+   // wait for comm to finish, if any
+   if (op.requests.size())
+   {
+      MPI_Waitall(op.requests.size(), op.requests.data(), MPI_STATUSES_IGNORE);
+      if (op.xghost_recv.Size())
+      {
+         // off-diagonal kernel
+         DerefineMatrixOpMultFunctor<Order, Atomic, false> func;
+         // directly read from host-pinned memory if not using GPU-aware MPI
+         func.xptr = Device::GetGPUAwareMPI() ? op.xghost_recv.Read()
+                     : op.xghost_recv.HostRead();
+         func.yptr = y.ReadWrite();
+         func.bsptr = op.block_storage.Read();
+         func.boptr = op.off_diag_block_offsets.Read();
+         func.brptr = op.block_off_diag_row_idcs_offsets.Read();
+         func.rsptr = op.recv_segment_idcs.Read();
+         func.segptr = op.recv_segments.Read();
+         func.coptr = op.block_off_diag_col_offsets.Read();
+         func.bwptr = op.block_off_diag_widths.Read();
+         func.rptr = op.row_off_diag_idcs.Read();
+         func.vdims = op.fespace->GetVDim();
+         func.nblocks = op.off_diag_block_offsets.Size();
+         func.width = op.xghost_recv.Size() / func.vdims;
+         func.height = op.Height() / func.vdims;
+         func.Run(op.max_rows);
+      }
+   }
+}
+} // namespace internal
+
+template <Ordering::Type Order, bool Atomic>
+ParDerefineMatrixOp::MultKernelType ParDerefineMatrixOp::MultKernel::Kernel()
+{
+   return internal::ParDerefMultKernelImpl<Order, Atomic>;
+}
+
+ParDerefineMatrixOp::MultKernelType
+ParDerefineMatrixOp::MultKernel::Fallback(Ordering::Type, bool)
+{
+   MFEM_ABORT("invalid MultKernel parameters");
+}
+
+ParDerefineMatrixOp::Kernels::Kernels()
+{
+   MultKernel::Specialization<Ordering::byNODES, false>::Add();
+   MultKernel::Specialization<Ordering::byVDIM, false>::Add();
+   MultKernel::Specialization<Ordering::byNODES, true>::Add();
+   MultKernel::Specialization<Ordering::byVDIM, true>::Add();
+}
+
+void ParDerefineMatrixOp::Mult(const Vector &x, Vector &y) const
+{
+   const bool is_dg = fespace->FEColl()->GetContType()
+                      == FiniteElementCollection::DISCONTINUOUS;
+   // DG needs atomic summation
+   MultKernel::Run(fespace->GetOrdering(), is_dg, *this, x, y);
+   // use this to prevent xghost* from being re-purposed for subsequent Mult
+   // calls
+   MFEM_DEVICE_SYNC;
+}
+
+ParDerefineMatrixOp::ParDerefineMatrixOp(ParFiniteElementSpace &fespace_,
+                                         int old_ndofs,
+                                         const Table *old_elem_dof,
+                                         const Table *old_elem_fos)
+   : Operator(fespace_.GetVSize(), old_ndofs * fespace_.GetVDim()),
+     fespace(&fespace_)
+{
+   static Kernels kernels;
+   constexpr int max_team_size = 256;
+
+   const int NRanks = fespace->GetNRanks();
+
+   const int nrk = HYPRE_AssumedPartitionCheck() ? 2 : NRanks;
+
+   MFEM_VERIFY(fespace->Nonconforming(),
+               "Not implemented for conforming meshes.");
+   MFEM_VERIFY(fespace->old_dof_offsets[nrk],
+               "Missing previous (finer) space.");
+
+   const int MyRank = fespace->GetMyRank();
+   ParNCMesh *old_pncmesh = fespace->GetParMesh()->pncmesh;
+   const CoarseFineTransformations &dtrans =
+      old_pncmesh->GetDerefinementTransforms();
+   const Array<int> &old_ranks = old_pncmesh->GetDerefineOldRanks();
+
+   const bool is_dg = fespace->FEColl()->GetContType()
+                      == FiniteElementCollection::DISCONTINUOUS;
+   DenseMatrix localRVO; // for variable-order only
+
+   DenseTensor localR[Geometry::NumGeom];
+   int diag_rows = 0;
+   int off_diag_rows = 0;
+   int diag_cols = 0;
+
+   auto get_ldofs = [&](int k) -> int
+   {
+      const Embedding &emb = dtrans.embeddings[k];
+      if (fespace->IsVariableOrder())
+      {
+         const FiniteElement *fe = fespace->GetFE(emb.parent);
+         return fe->GetDof();
+      }
+      else
+      {
+         Geometry::Type geom =
+         fespace->GetParMesh()->GetElementBaseGeometry(emb.parent);
+         return fespace->FEColl()->FiniteElementForGeometry(geom)->GetDof();
+      }
+   };
+   Array<int> dofs, old_dofs;
+   max_rows = 1;
+   // first pass:
+   // - determine memory block lengths
+   // - identify dofs in x we need to send/receive
+   // don't need to send the indices, fine rank will re-arrange and sign
+   // change x before transmitting the ghost data
+
+   // key: coarse rank to send to
+   // value: old dofs to send (with sign)
+   std::map<int, std::vector<int>> to_send;
+   // key: fine rank
+   // value: indices into dtrans.embeddings
+   std::map<int, std::vector<int>> od_ks;
+   // key: fine rank
+   // value: recv segment length
+   std::map<int, int> od_seg_lens;
+   int send_len = 0;
+   int recv_len = 0;
+   // size of block_storage, if fespace->IsVariableOrder()
+   // otherwise unused
+   int total_size = 0;
+   int num_diagonal_blocks = 0;
+   int num_offdiagonal_blocks = 0;
+   for (int k = 0; k < dtrans.embeddings.Size(); ++k)
+   {
+      const Embedding &emb = dtrans.embeddings[k];
+      int fine_rank = old_ranks[k];
+      int coarse_rank = (emb.parent < 0) ? (-1 - emb.parent)
+                        : old_pncmesh->ElementRank(emb.parent);
+      if (coarse_rank != MyRank && fine_rank == MyRank)
+      {
+         // this rank needs to send data in x to course_rank
+         old_elem_dof->GetRow(k, old_dofs);
+         auto &tmp = to_send[coarse_rank];
+         send_len += old_dofs.Size();
+         for (int i = 0; i < old_dofs.Size(); ++i)
+         {
+            tmp.emplace_back(old_dofs[i]);
+         }
+      }
+      else if (coarse_rank == MyRank && fine_rank != MyRank)
+      {
+         // this rank needs to receive data in x from fine_rank
+         MFEM_ASSERT(emb.parent >= 0, "");
+         auto ldofs = get_ldofs(k);
+         off_diag_rows += ldofs;
+         recv_len += ldofs;
+         od_ks[fine_rank].emplace_back(k);
+         od_seg_lens[fine_rank] += ldofs;
+         ++num_offdiagonal_blocks;
+         if (fespace->IsVariableOrder())
+         {
+            total_size += ldofs * ldofs;
+         }
+      }
+      else if (coarse_rank == MyRank && fine_rank == MyRank)
+      {
+         MFEM_ASSERT(emb.parent >= 0, "");
+         // diagonal
+         ++num_diagonal_blocks;
+         auto ldofs = get_ldofs(k);
+         diag_rows += ldofs;
+         diag_cols += ldofs;
+         if (fespace->IsVariableOrder())
+         {
+            total_size += ldofs * ldofs;
+         }
+      }
+   }
+   send_segments.SetSize(to_send.size() + 1);
+   send_segments.HostWrite();
+   send_ranks.SetSize(to_send.size());
+   send_ranks.HostWrite();
+   {
+      int idx = 0;
+      send_segments[0] = 0;
+      for (auto &tmp : to_send)
+      {
+         send_ranks[idx] = tmp.first;
+         send_segments[idx + 1] = send_segments[idx] + tmp.second.size();
+         ++idx;
+      }
+   }
+   recv_segment_idcs.SetSize(off_diag_rows);
+   recv_segment_idcs.HostWrite();
+   recv_segments.SetSize(od_ks.size() + 1);
+   recv_segments.HostWrite();
+   recv_ranks.SetSize(od_ks.size());
+   recv_ranks.HostWrite();
+
+   // set sizes
+   row_idcs.SetSize(diag_rows);
+   row_idcs.HostWrite();
+   row_off_diag_idcs.SetSize(off_diag_rows);
+   row_off_diag_idcs.HostWrite();
+   col_idcs.SetSize(diag_cols);
+   col_idcs.HostWrite();
+   block_row_idcs_offsets.SetSize(num_diagonal_blocks + 1);
+   block_row_idcs_offsets.HostWrite();
+   block_col_idcs_offsets.SetSize(num_diagonal_blocks + 1);
+   block_col_idcs_offsets.HostWrite();
+   block_off_diag_row_idcs_offsets.SetSize(num_offdiagonal_blocks + 1);
+   block_off_diag_row_idcs_offsets.HostWrite();
+   block_off_diag_col_offsets.SetSize(num_offdiagonal_blocks);
+   block_off_diag_col_offsets.HostWrite();
+   block_off_diag_widths.SetSize(num_offdiagonal_blocks);
+   block_off_diag_widths.HostWrite();
+   pack_col_idcs.SetSize(send_len);
+   // memory manager doesn't appear to have a graceful fallback for
+   // HOST_PINNED if not built with CUDA or HIP
+#if defined(MFEM_USE_CUDA) or defined(MFEM_USE_HIP)
+   xghost_send.SetSize(send_len * fespace->GetVDim(),
+                       Device::GetGPUAwareMPI() ? MemoryType::DEFAULT
+                       : MemoryType::HOST_PINNED);
+   xghost_recv.SetSize(recv_len * fespace->GetVDim(),
+                       Device::GetGPUAwareMPI() ? MemoryType::DEFAULT
+                       : MemoryType::HOST_PINNED);
+#else
+   xghost_send.SetSize(send_len * fespace->GetVDim());
+   xghost_recv.SetSize(recv_len * fespace->GetVDim());
+#endif
+   send_permutations.SetSize(send_len);
+   send_segment_idcs.SetSize(send_len);
+   block_offsets.SetSize(num_diagonal_blocks);
+   block_offsets.HostWrite();
+   off_diag_block_offsets.SetSize(num_offdiagonal_blocks);
+   off_diag_block_offsets.HostWrite();
+   int geom_offsets[Geometry::NumGeom];
+   real_t *bs_ptr;
+
+   if (fespace->IsVariableOrder())
+   {
+      block_storage.SetSize(total_size);
+      bs_ptr = block_storage.HostWrite();
+      // compute block data later
+   }
+   else
+   {
+      // compression scheme:
+      // block_offsets is the start of each block, potentially repeated
+      // only need to store localR for used shapes
+      Mesh::GeometryList elem_geoms(*fespace->GetMesh());
+
+      int size = 0;
+      for (int i = 0; i < elem_geoms.Size(); ++i)
+      {
+         fespace->GetLocalDerefinementMatrices(elem_geoms[i],
+                                               localR[elem_geoms[i]]);
+         geom_offsets[elem_geoms[i]] = size;
+         size += localR[elem_geoms[i]].TotalSize();
+      }
+      block_storage.SetSize(size);
+      bs_ptr = block_storage.HostWrite();
+      // copy blocks into block_storage
+      for (int i = 0; i < elem_geoms.Size(); ++i)
+      {
+         std::copy(localR[elem_geoms[i]].Data(),
+                   localR[elem_geoms[i]].Data()
+                   + localR[elem_geoms[i]].TotalSize(),
+                   bs_ptr);
+         bs_ptr += localR[elem_geoms[i]].TotalSize();
+      }
+   }
+
+   // second pass:
+   // - initialize buffers
+
+   {
+      auto ptr = send_permutations.HostWrite();
+      auto ptr2 = send_segment_idcs.HostWrite();
+      int i = 0;
+      for (auto &v : to_send)
+      {
+         ptr = std::copy(v.second.begin(), v.second.end(), ptr);
+         for (size_t idx = 0; idx < v.second.size(); ++idx)
+         {
+            *ptr2 = i;
+            ++ptr2;
+         }
+         ++i;
+      }
+   }
+
+   block_row_idcs_offsets[0] = 0;
+   block_col_idcs_offsets[0] = 0;
+   block_off_diag_row_idcs_offsets[0] = 0;
+   Array<int> mark(fespace->GetNDofs());
+   mark = 0;
+   {
+      int idx = 0;
+      recv_segments[0] = 0;
+      for (auto &v : od_seg_lens)
+      {
+         recv_ranks[idx] = v.first;
+         recv_segments[idx + 1] = recv_segments[idx] + v.second;
+         ++idx;
+      }
+   }
+   // key: index into dtrans.embeddings
+   // value: off-diagonal block offset, od_ridx, seg id
+   std::unordered_map<int, std::array<int, 3>> ks_map;
+   {
+      int od_ridx = 0;
+      int seg_id = 0;
+      for (auto &v1 : od_ks)
+      {
+         for (auto k : v1.second)
+         {
+            auto &tmp = ks_map[k];
+            tmp[0] = ks_map.size() - 1;
+            tmp[1] = od_ridx;
+            tmp[2] = seg_id;
+            od_ridx += get_ldofs(k);
+         }
+         ++seg_id;
+      }
+   }
+   int diag_idx = 0;
+   int var_offset = 0;
+   int ridx = 0;
+   int cidx = 0;
+   // can't break this up into separate diagonals/off-diagonals loops because
+   // of mark
+   for (int k = 0; k < dtrans.embeddings.Size(); ++k)
+   {
+      const Embedding &emb = dtrans.embeddings[k];
+      if (emb.parent < 0)
+      {
+         continue;
+      }
+      int fine_rank = old_ranks[k];
+      int coarse_rank = (emb.parent < 0) ? (-1 - emb.parent)
+                        : old_pncmesh->ElementRank(emb.parent);
+      if (coarse_rank == MyRank)
+      {
+         // either diagonal or off-diagonal
+         Geometry::Type geom =
+            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
+         if (fespace->IsVariableOrder())
+         {
+            const FiniteElement *fe = fespace->GetFE(emb.parent);
+            const DenseTensor &pmats = dtrans.point_matrices[geom];
+            const int ldof = fe->GetDof();
+
+            IsoparametricTransformation isotr;
+            isotr.SetIdentityTransformation(geom);
+
+            localRVO.SetSize(ldof, ldof);
+            isotr.SetPointMat(pmats(emb.matrix));
+            // Local restriction is size ldofxldof assuming that the parent
+            // and child are of same polynomial order.
+            fe->GetLocalRestriction(isotr, localRVO);
+            // copy block
+            auto s = localRVO.Height() * localRVO.Width();
+            std::copy(localRVO.Data(), localRVO.Data() + s, bs_ptr);
+            bs_ptr += s;
+         }
+         DenseMatrix &lR =
+            fespace->IsVariableOrder() ? localRVO : localR[geom](emb.matrix);
+         max_rows = std::max(lR.Height(), max_rows);
+         auto size = lR.Height() * lR.Width();
+         fespace->elem_dof->GetRow(emb.parent, dofs);
+         if (fine_rank == MyRank)
+         {
+            // diagonal
+            old_elem_dof->GetRow(k, old_dofs);
+            MFEM_VERIFY(old_dofs.Size() == dofs.Size(),
+                        "Parent and child must have same #dofs.");
+            block_row_idcs_offsets[diag_idx + 1] =
+               block_row_idcs_offsets[diag_idx] + lR.Height();
+            block_col_idcs_offsets[diag_idx + 1] =
+               block_col_idcs_offsets[diag_idx] + lR.Width();
+
+            if (fespace->IsVariableOrder())
+            {
+               block_offsets[diag_idx] = var_offset;
+               var_offset += size;
+            }
+            else
+            {
+               block_offsets[diag_idx] = geom_offsets[geom] + size * emb.matrix;
+            }
+            for (int i = 0; i < lR.Height(); ++i, ++ridx)
+            {
+               if (!std::isfinite(lR(i, 0)))
+               {
+                  row_idcs[ridx] = INT_MAX;
+                  continue;
+               }
+               int r = dofs[i];
+               int m = (r >= 0) ? r : (-1 - r);
+               if (is_dg || !mark[m])
+               {
+                  row_idcs[ridx] = r;
+                  mark[m] = 1;
+               }
+               else
+               {
+                  row_idcs[ridx] = INT_MAX;
+               }
+            }
+            for (int i = 0; i < lR.Width(); ++i, ++cidx)
+            {
+               col_idcs[cidx] = old_dofs[i];
+            }
+            ++diag_idx;
+         }
+         else
+         {
+            // off-diagonal
+            auto &tmp = ks_map.at(k);
+            auto od_idx = tmp[0];
+            auto od_ridx = tmp[1];
+            block_off_diag_row_idcs_offsets[od_idx + 1] =
+               block_off_diag_row_idcs_offsets[od_idx] + lR.Height();
+            block_off_diag_col_offsets[od_idx] = od_ridx;
+            block_off_diag_widths[od_idx] = lR.Width();
+            recv_segment_idcs[od_idx] = tmp[2];
+
+            if (fespace->IsVariableOrder())
+            {
+               off_diag_block_offsets[od_idx] = var_offset;
+               var_offset += size;
+            }
+            else
+            {
+               off_diag_block_offsets[od_idx] =
+                  geom_offsets[geom] + size * emb.matrix;
+            }
+            for (int i = 0; i < lR.Height(); ++i, ++od_ridx)
+            {
+               if (!std::isfinite(lR(i, 0)))
+               {
+                  row_off_diag_idcs[od_ridx] = INT_MAX;
+                  continue;
+               }
+               int r = dofs[i];
+               int m = (r >= 0) ? r : (-1 - r);
+               if (is_dg || !mark[m])
+               {
+                  row_off_diag_idcs[od_ridx] = r;
+                  mark[m] = 1;
+               }
+               else
+               {
+                  row_off_diag_idcs[od_ridx] = INT_MAX;
+               }
+            }
+            ++od_idx;
+         }
+      }
+   }
+   // if not using GPU, set max_rows/max_cols to zero
+   if (Device::Allows(Backend::DEVICE_MASK))
+   {
+      max_rows = std::min(max_rows, max_team_size);
+   }
+   else
+   {
+      max_rows = 1;
+   }
+   requests.reserve(recv_ranks.Size() + send_ranks.Size());
+}
+} // namespace mfem
+/// \endcond DO_NOT_DOCUMENT
+
+#endif

fem/pderefmat_op.hpp

@@ -0,0 +1,111 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_PDEREFMAT_OP
+#define MFEM_PDEREFMAT_OP
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "pfespace.hpp"
+
+#include "kernel_dispatch.hpp"
+
+#include <vector>
+
+namespace mfem
+{
+
+/// \cond DO_NOT_DOCUMENT
+struct ParDerefineMatrixOp : public Operator
+{
+   ParFiniteElementSpace *fespace;
+   /// offsets into block_storage for diagonal
+   Array<int> block_offsets;
+   /// offsets into row_idcs for diagonal
+   Array<int> block_row_idcs_offsets;
+   /// offsets into col_idcs for diagonal
+   Array<int> block_col_idcs_offsets;
+
+   /// offsets into block_storage for off-diagonal
+   Array<int> off_diag_block_offsets;
+   /// offsets into row_idcs for off-diagonal
+   Array<int> block_off_diag_row_idcs_offsets;
+   Array<int> block_off_diag_col_offsets;
+   Array<int> block_off_diag_widths;
+   /// mapping for row dofs, INT_MAX indicates the block row should be ignored.
+   /// negative means the row data should be negated.
+   /// only for diagonal blocks
+   Array<int> row_idcs;
+   /// mapping for col dofs, negative means the col data should be negated.
+   /// only for diagonal blocks
+   Array<int> col_idcs;
+
+   Array<int> pack_col_idcs;
+
+   /// mapping for row dofs, INT_MAX indicates the block row should be ignored.
+   /// negative means the row data should be negated.
+   /// only for off-diagonal blocks
+   Array<int> row_off_diag_idcs;
+   /// dense block matrices which can be reused to construct the full matrix
+   /// operation. These are stored contiguously and blocks have no restrictions
+   /// on shape (can be rectangle and differ from block to block).
+   /// This is only for the diagonal block.
+   Vector block_storage;
+   /// maximum height of any block in block_storage for GPU
+   /// parallelization, or 1 for CPU runs.
+   int max_rows;
+
+   /// quasi Ordering::byNODES, broken into sections by ranks we need to send
+   /// the data to
+   mutable Vector xghost_send;
+   /// quasi Ordering::byNODES, broken into sections by ranks we received
+   /// the data from
+   mutable Vector xghost_recv;
+   /// maps off-diagonal k to segment
+   Array<int> recv_segment_idcs;
+   /// cumulative count of dofs which will be received from other ranks
+   Array<int> recv_segments;
+   /// Source rank of each recv segment
+   Array<int> recv_ranks;
+   /// What send segment each entry in send_permutations corresponds to
+   Array<int> send_segment_idcs;
+   /// cumulative count of dofs which will be sent to other ranks
+   Array<int> send_segments;
+   /// Destination rank of each send segment
+   Array<int> send_ranks;
+   /// how to permute/sign change values from our local x to send to other ranks
+   Array<int> send_permutations;
+   /// internal buffer for MPI requests
+   mutable std::vector<MPI_Request> requests;
+
+   using MultKernelType = void (*)(const ParDerefineMatrixOp &, const Vector &,
+                                   Vector &);
+   /// template args: ordering, atomic
+   MFEM_REGISTER_KERNELS(MultKernel, MultKernelType, (Ordering::Type, bool));
+
+   struct Kernels
+   {
+      Kernels();
+   };
+
+   void Mult(const Vector &x, Vector &y) const;
+
+   ParDerefineMatrixOp(ParFiniteElementSpace &fespace_, int old_ndofs,
+                       const Table *old_elem_dof, const Table *old_elem_fos);
+};
+/// \endcond DO_NOT_DOCUMENT
+} // namespace mfem
+
+#endif
+
+#endif

fem/pfespace.cpp

@@ -22,12 +22,13 @@
 #include "../mesh/mesh_headers.hpp"
 #include "../general/binaryio.hpp"
 
+#include "pderefmat_op.hpp"
+
 #include <limits>
 #include <list>
 
 namespace mfem
 {
-
 ParFiniteElementSpace::ParFiniteElementSpace(
    const ParFiniteElementSpace &orig, ParMesh *pmesh,
    const FiniteElementCollection *fec)
@@ -4487,13 +4488,6 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
    return M;
 }
 
-
-struct DerefDofMessage
-{
-   std::vector<HYPRE_BigInt> dofs;
-   MPI_Request request;
-};
-
 HypreParMatrix*
 ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
                                                   const Table* old_elem_dof,
@@ -4536,7 +4530,13 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
       old_pncmesh->GetDerefinementTransforms();
    const Array<int> &old_ranks = old_pncmesh->GetDerefineOldRanks();
 
-   std::map<int, DerefDofMessage> messages;
+   // key: other rank
+   // value: send or recieve buffer
+   std::map<int, std::vector<HYPRE_BigInt>> to_send;
+   std::map<int, std::vector<HYPRE_BigInt>> to_recv;
+   // key: index into dtrans.embeddings
+   // value: [start, stop]
+   std::unordered_map<int, std::array<size_t, 2>> recv_messages;
 
    HYPRE_BigInt old_offset = HYPRE_AssumedPartitionCheck()
                              ? old_dof_offsets[0] : old_dof_offsets[MyRank];
@@ -4556,30 +4556,46 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
          old_elem_dof->GetRow(k, dofs);
          DofsToVDofs(dofs, old_ndofs);
 
-         DerefDofMessage &msg = messages[k];
-         msg.dofs.resize(dofs.Size());
+         std::vector<HYPRE_BigInt>& send_buf = to_send[coarse_rank];
+         auto pos = send_buf.size();
+         send_buf.resize(pos + dofs.Size());
          for (int i = 0; i < dofs.Size(); i++)
          {
-            msg.dofs[i] = old_offset + dofs[i];
+            send_buf[pos + i] = old_offset + dofs[i];
          }
-
-         MPI_Isend(&msg.dofs[0], static_cast<int>(msg.dofs.size()), HYPRE_MPI_BIG_INT,
-                   coarse_rank, 291, MyComm, &msg.request);
       }
       else if (coarse_rank == MyRank && fine_rank != MyRank)
       {
          MFEM_ASSERT(emb.parent >= 0, "");
          Geometry::Type geom = mesh->GetElementBaseGeometry(emb.parent);
 
-         DerefDofMessage &msg = messages[k];
-         msg.dofs.resize(ldof[geom]*vdim);
-
-         MPI_Irecv(&msg.dofs[0], ldof[geom]*vdim, HYPRE_MPI_BIG_INT,
-                   fine_rank, 291, MyComm, &msg.request);
+         std::vector<HYPRE_BigInt>& recv_buf = to_recv[fine_rank];
+         auto& msg = recv_messages[k];
+         msg[0] = recv_buf.size();
+         recv_buf.resize(recv_buf.size() + ldof[geom] * vdim);
+         msg[1] = recv_buf.size();
       }
-      // TODO: coalesce Isends/Irecvs to the same rank. Typically, on uniform
-      // derefinement, there should be just one send to MyRank-1 and one recv
-      // from MyRank+1
+   }
+
+   // assume embedding orders are consistent (i.e. what we expect to receive
+   // first from a given rank is sent first, etc.)
+   std::vector<MPI_Request> requests;
+   requests.reserve(to_send.size() + to_recv.size());
+   // enqueue recvs
+   for (auto &v : to_recv)
+   {
+      requests.emplace_back();
+      MPI_Irecv(v.second.data(), v.second.size(), HYPRE_MPI_BIG_INT, v.first,
+                MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA, MyComm,
+                &requests.back());
+   }
+   // enqueue sends
+   for (auto &v : to_send)
+   {
+      requests.emplace_back();
+      MPI_Isend(v.second.data(), v.second.size(), HYPRE_MPI_BIG_INT, v.first,
+                MessageTag::DEREFINEMENT_MATRIX_CONSTRUCTION_DATA, MyComm,
+                &requests.back());
    }
 
    DenseTensor localR[Geometry::NumGeom];
@@ -4637,10 +4653,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    diag->Finalize();
 
    // wait for all sends/receives to complete
-   for (auto it = messages.begin(); it != messages.end(); ++it)
-   {
-      MPI_Wait(&it->second.request, MPI_STATUS_IGNORE);
-   }
+   MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
 
    // create the off-diagonal part of the derefinement matrix
    SparseMatrix *offd = new SparseMatrix(ndofs*vdim, 1);
@@ -4661,13 +4674,14 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
 
          elem_dof->GetRow(emb.parent, dofs);
 
-         DerefDofMessage &msg = messages[k];
-         MFEM_ASSERT(msg.dofs.size(), "");
+         auto& odofs = to_recv.at(fine_rank);
+         auto &msg = recv_messages[k];
+         MFEM_ASSERT(msg[1] > msg[0], "");
 
          for (int vd = 0; vd < vdim; vd++)
          {
             MFEM_ASSERT(ldof[geom], "");
-            HYPRE_BigInt* remote_dofs = &msg.dofs[vd*ldof[geom]];
+            HYPRE_BigInt *remote_dofs = odofs.data() + msg[0] + vd * ldof[geom];
 
             for (int i = 0; i < lR.Height(); i++)
             {
@@ -4694,7 +4708,6 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
       }
    }
 
-   messages.clear();
    offd->Finalize(0);
    offd->SetWidth(static_cast<int>(col_map.size()));
 
@@ -4946,8 +4959,13 @@ void ParFiniteElementSpace::Update(bool want_transform)
 
          case Mesh::DEREFINE:
          {
+#if 0
             Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof,
                                                 old_elem_fos));
+#else
+            Th.Reset(new ParDerefineMatrixOp(*this, old_ndofs, old_elem_dof,
+                                             old_elem_fos));
+#endif
             if (Nonconforming())
             {
                Th.SetOperatorOwner(false);
@@ -5259,7 +5277,7 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
       gc.GetNeighborLTDofTable(nbr_ltdof);
       const int nb_connections = nbr_ltdof.Size_of_connections();
       shr_ltdof.SetSize(nb_connections);
-      shr_ltdof.CopyFrom(nbr_ltdof.GetJ());
+      if (nb_connections > 0) { shr_ltdof.CopyFrom(nbr_ltdof.GetJ()); }
       shr_buf.SetSize(nb_connections);
       shr_buf.UseDevice(true);
       shr_buf_offsets = nbr_ltdof.GetIMemory();
@@ -5288,7 +5306,7 @@ DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
       gc.GetNeighborLDofTable(nbr_ldof);
       const int nb_connections = nbr_ldof.Size_of_connections();
       ext_ldof.SetSize(nb_connections);
-      ext_ldof.CopyFrom(nbr_ldof.GetJ());
+      if (nb_connections > 0) { ext_ldof.CopyFrom(nbr_ldof.GetJ()); }
       ext_ldof.GetMemory().UseDevice(true);
       ext_buf.SetSize(nb_connections);
       ext_buf.UseDevice(true);

fem/pfespace.hpp

@@ -24,9 +24,12 @@
 namespace mfem
 {
 
+struct ParDerefineMatrixOp;
+
 /// Abstract parallel finite element space.
 class ParFiniteElementSpace : public FiniteElementSpace
 {
+   friend struct ParDerefineMatrixOp;
 private:
    /// MPI data.
    MPI_Comm MyComm;

fem/qfunction.hpp

@@ -214,16 +214,16 @@ public:
 inline void QuadratureFunction::GetValues(
    int idx, Vector &values)
 {
-   const int s_offset = qspace->offsets[idx];
-   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   const int s_offset = qspace->Offset(idx);
+   const int sl_size = qspace->Offset(idx + 1) - s_offset;
    values.MakeRef(*this, vdim*s_offset, vdim*sl_size);
 }
 
 inline void QuadratureFunction::GetValues(
    int idx, Vector &values) const
 {
-   const int s_offset = qspace->offsets[idx];
-   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   const int s_offset = qspace->Offset(idx);
+   const int sl_size = qspace->Offset(idx + 1) - s_offset;
    values.SetSize(vdim*sl_size);
    values.HostWrite();
    const real_t *q = HostRead() + vdim*s_offset;
@@ -236,14 +236,14 @@ inline void QuadratureFunction::GetValues(
 inline void QuadratureFunction::GetValues(
    int idx, const int ip_num, Vector &values)
 {
-   const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim;
+   const int s_offset = qspace->Offset(idx) * vdim + ip_num * vdim;
    values.MakeRef(*this, s_offset, vdim);
 }
 
 inline void QuadratureFunction::GetValues(
    int idx, const int ip_num, Vector &values) const
 {
-   const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim;
+   const int s_offset = qspace->Offset(idx) * vdim + ip_num * vdim;
    values.SetSize(vdim);
    values.HostWrite();
    const real_t *q = HostRead() + s_offset;
@@ -256,8 +256,8 @@ inline void QuadratureFunction::GetValues(
 inline void QuadratureFunction::GetValues(
    int idx, DenseMatrix &values)
 {
-   const int s_offset = qspace->offsets[idx];
-   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   const int s_offset = qspace->Offset(idx);
+   const int sl_size = qspace->Offset(idx + 1) - s_offset;
    // Make the values matrix memory an alias of the quadrature function memory
    Memory<real_t> &values_mem = values.GetMemory();
    values_mem.Delete();
@@ -268,8 +268,8 @@ inline void QuadratureFunction::GetValues(
 inline void QuadratureFunction::GetValues(
    int idx, DenseMatrix &values) const
 {
-   const int s_offset = qspace->offsets[idx];
-   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   const int s_offset = qspace->Offset(idx);
+   const int sl_size = qspace->Offset(idx + 1) - s_offset;
    values.SetSize(vdim, sl_size);
    values.HostWrite();
    const real_t *q = HostRead() + vdim*s_offset;

fem/qinterp/det.cpp

@@ -9,278 +9,16 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../quadinterpolator.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../fem/kernels.hpp"
-#include "../../linalg/kernels.hpp"
-
-using namespace mfem;
+#include "det.hpp"
 
 namespace mfem
 {
 
 namespace internal
 {
-
 namespace quadrature_interpolator
 {
 
-static void Det1D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d,
-                  const int q1d,
-                  Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(b);
-   MFEM_CONTRACT_VAR(d_buff);
-   const auto G = Reshape(g, q1d, d1d);
-   const auto X = Reshape(x, d1d, NE);
-
-   auto Y = Reshape(y, q1d, NE);
-
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
-   {
-      for (int q = 0; q < q1d; q++)
-      {
-         real_t u = 0.0;
-         for (int d = 0; d < d1d; d++)
-         {
-            u += G(q, d) * X(d, e);
-         }
-         Y(q, e) = u;
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void Det2D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d = 0,
-                  const int q1d = 0,
-                  Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(d_buff);
-   static constexpr int SDIM = 2;
-   static constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, NE);
-
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t XY[SDIM][NBZ][MD1*MD1];
-      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
-      MFEM_SHARED real_t QQ[2*SDIM][NBZ][MQ1*MQ1];
-
-      kernels::internal::LoadX<MD1,NBZ>(e,D1D,X,XY);
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::internal::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
-      kernels::internal::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            real_t J[4];
-            kernels::internal::PullGrad<MQ1,NBZ>(Q1D,qx,qy,QQ,J);
-            Y(qx,qy,e) = kernels::Det<2>(J);
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void Det2DSurface(const int NE,
-                         const real_t *b,
-                         const real_t *g,
-                         const real_t *x,
-                         real_t *y,
-                         const int d1d = 0,
-                         const int q1d = 0,
-                         Vector *d_buff = nullptr)
-{
-   MFEM_CONTRACT_VAR(d_buff);
-
-   static constexpr int SDIM = 3;
-   static constexpr int NBZ = 1;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, NE);
-
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int tidz = MFEM_THREAD_ID(z);
-
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t XYZ[SDIM][NBZ][MD1*MD1];
-      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
-
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      // Load XYZ components
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int d = 0; d < SDIM; ++d)
-            {
-               XYZ[d][tidz][dx + dy*D1D] = X(dx,dy,d,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      ConstDeviceMatrix B_mat(BG[0], D1D, Q1D);
-      ConstDeviceMatrix G_mat(BG[1], D1D, Q1D);
-
-      // x contraction
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            for (int d = 0; d < SDIM; ++d)
-            {
-               real_t u = 0.0;
-               real_t v = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t xval = XYZ[d][tidz][dx + dy*D1D];
-                  u += xval * G_mat(dx,qx);
-                  v += xval * B_mat(dx,qx);
-               }
-               DQ[d][tidz][dy + qx*D1D] = u;
-               DQ[3 + d][tidz][dy + qx*D1D] = v;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // y contraction and determinant computation
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            real_t J_[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
-            for (int d = 0; d < SDIM; ++d)
-            {
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  J_[d] += DQ[d][tidz][dy + qx*D1D] * B_mat(dy,qy);
-                  J_[3 + d] += DQ[3 + d][tidz][dy + qx*D1D] * G_mat(dy,qy);
-               }
-            }
-            DeviceTensor<2> J(J_, 3, 2);
-            const real_t E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
-            const real_t F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
-            const real_t G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
-            Y(qx,qy,e) = std::sqrt(E*G - F*F);
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, bool SMEM = true>
-static void Det3D(const int NE,
-                  const real_t *b,
-                  const real_t *g,
-                  const real_t *x,
-                  real_t *y,
-                  const int d1d = 0,
-                  const int q1d = 0,
-                  Vector *d_buff = nullptr) // used only with SMEM = false
-{
-   constexpr int DIM = 3;
-   static constexpr int GRID = SMEM ? 0 : 128;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
-   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
-
-   real_t *GM = nullptr;
-   if (!SMEM)
-   {
-      const DeviceDofQuadLimits &limits = DeviceDofQuadLimits::Get();
-      const int max_q1d = T_Q1D ? T_Q1D : limits.MAX_Q1D;
-      const int max_d1d = T_D1D ? T_D1D : limits.MAX_D1D;
-      const int max_qd = std::max(max_q1d, max_d1d);
-      const int mem_size = max_qd * max_qd * max_qd * 9;
-      d_buff->SetSize(2*mem_size*GRID);
-      GM = d_buff->Write();
-   }
-
-   mfem::forall_3D_grid(NE, Q1D, Q1D, Q1D, GRID, [=] MFEM_HOST_DEVICE (int e)
-   {
-      static constexpr int MQ1 = T_Q1D ? T_Q1D :
-                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_Q1D);
-      static constexpr int MD1 = T_D1D ? T_D1D :
-                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_D1D);
-      static constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
-      static constexpr int MSZ = MDQ * MDQ * MDQ * 9;
-
-      const int bid = MFEM_BLOCK_ID(x);
-      MFEM_SHARED real_t BG[2][MQ1*MD1];
-      MFEM_SHARED real_t SM0[SMEM?MSZ:1];
-      MFEM_SHARED real_t SM1[SMEM?MSZ:1];
-      real_t *lm0 = SMEM ? SM0 : GM + MSZ*bid;
-      real_t *lm1 = SMEM ? SM1 : GM + MSZ*(GRID+bid);
-      real_t (*DDD)[MD1*MD1*MD1] = (real_t (*)[MD1*MD1*MD1]) (lm0);
-      real_t (*DDQ)[MD1*MD1*MQ1] = (real_t (*)[MD1*MD1*MQ1]) (lm1);
-      real_t (*DQQ)[MD1*MQ1*MQ1] = (real_t (*)[MD1*MQ1*MQ1]) (lm0);
-      real_t (*QQQ)[MQ1*MQ1*MQ1] = (real_t (*)[MQ1*MQ1*MQ1]) (lm1);
-
-      kernels::internal::LoadX<MD1>(e,D1D,X,DDD);
-      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
-
-      kernels::internal::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
-      kernels::internal::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
-      kernels::internal::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
-
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               real_t J[9];
-               kernels::internal::PullGrad<MQ1>(Q1D, qx,qy,qz, QQQ, J);
-               Y(qx,qy,qz,e) = kernels::Det<3>(J);
-            }
-         }
-      }
-   });
-}
-
 void InitDetKernels()
 {
    using k = QuadratureInterpolator::DetKernels;
@@ -302,27 +40,12 @@ void InitDetKernels()
 }
 
 } // namespace quadrature_interpolator
-
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-namespace
-{
-using DetKernel = QuadratureInterpolator::DetKernelType;
-}
-
-template<int DIM, int SDIM, int D1D, int Q1D>
-DetKernel QuadratureInterpolator::DetKernels::Kernel()
-{
-   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
-   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
-   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
-   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
-}
-
-DetKernel QuadratureInterpolator::DetKernels::Fallback(
+QuadratureInterpolator::DetKernelType
+QuadratureInterpolator::DetKernels::Fallback(
    int DIM, int SDIM, int D1D, int Q1D)
 {
    if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }

fem/qinterp/det.hpp

@@ -0,0 +1,304 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_QUADINTERP_DET_HPP
+#define MFEM_QUADINTERP_DET_HPP
+
+#include "../quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../fem/kernels.hpp"
+#include "../../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+namespace quadrature_interpolator
+{
+
+inline void Det1D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d,
+                  const int q1d,
+                  Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(b);
+   MFEM_CONTRACT_VAR(d_buff);
+   const auto G = Reshape(g, q1d, d1d);
+   const auto X = Reshape(x, d1d, NE);
+
+   auto Y = Reshape(y, q1d, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int q = 0; q < q1d; q++)
+      {
+         real_t u = 0.0;
+         for (int d = 0; d < d1d; d++)
+         {
+            u += G(q, d) * X(d, e);
+         }
+         Y(q, e) = u;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void Det2D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d = 0,
+                  const int q1d = 0,
+                  Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(d_buff);
+   static constexpr int SDIM = 2;
+   static constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, NE);
+
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t XY[SDIM][NBZ][MD1*MD1];
+      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
+      MFEM_SHARED real_t QQ[2*SDIM][NBZ][MQ1*MQ1];
+
+      kernels::internal::LoadX<MD1,NBZ>(e,D1D,X,XY);
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::internal::GradX<MD1,MQ1,NBZ>(D1D,Q1D,BG,XY,DQ);
+      kernels::internal::GradY<MD1,MQ1,NBZ>(D1D,Q1D,BG,DQ,QQ);
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            real_t J[4];
+            kernels::internal::PullGrad<MQ1,NBZ>(Q1D,qx,qy,QQ,J);
+            Y(qx,qy,e) = kernels::Det<2>(J);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void Det2DSurface(const int NE,
+                         const real_t *b,
+                         const real_t *g,
+                         const real_t *x,
+                         real_t *y,
+                         const int d1d = 0,
+                         const int q1d = 0,
+                         Vector *d_buff = nullptr)
+{
+   MFEM_CONTRACT_VAR(d_buff);
+
+   static constexpr int SDIM = 3;
+   static constexpr int NBZ = 1;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x,  D1D, D1D, SDIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, NE);
+
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int tidz = MFEM_THREAD_ID(z);
+
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t XYZ[SDIM][NBZ][MD1*MD1];
+      MFEM_SHARED real_t DQ[2*SDIM][NBZ][MD1*MQ1];
+
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      // Load XYZ components
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int d = 0; d < SDIM; ++d)
+            {
+               XYZ[d][tidz][dx + dy*D1D] = X(dx,dy,d,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      ConstDeviceMatrix B_mat(BG[0], D1D, Q1D);
+      ConstDeviceMatrix G_mat(BG[1], D1D, Q1D);
+
+      // x contraction
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            for (int d = 0; d < SDIM; ++d)
+            {
+               real_t u = 0.0;
+               real_t v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t xval = XYZ[d][tidz][dx + dy*D1D];
+                  u += xval * G_mat(dx,qx);
+                  v += xval * B_mat(dx,qx);
+               }
+               DQ[d][tidz][dy + qx*D1D] = u;
+               DQ[3 + d][tidz][dy + qx*D1D] = v;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // y contraction and determinant computation
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            real_t J_[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
+            for (int d = 0; d < SDIM; ++d)
+            {
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  J_[d] += DQ[d][tidz][dy + qx*D1D] * B_mat(dy,qy);
+                  J_[3 + d] += DQ[3 + d][tidz][dy + qx*D1D] * G_mat(dy,qy);
+               }
+            }
+            DeviceTensor<2> J(J_, 3, 2);
+            const real_t E = J(0,0)*J(0,0) + J(1,0)*J(1,0) + J(2,0)*J(2,0);
+            const real_t F = J(0,0)*J(0,1) + J(1,0)*J(1,1) + J(2,0)*J(2,1);
+            const real_t G = J(0,1)*J(0,1) + J(1,1)*J(1,1) + J(2,1)*J(2,1);
+            Y(qx,qy,e) = std::sqrt(E*G - F*F);
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, bool SMEM = true>
+inline void Det3D(const int NE,
+                  const real_t *b,
+                  const real_t *g,
+                  const real_t *x,
+                  real_t *y,
+                  const int d1d = 0,
+                  const int q1d = 0,
+                  Vector *d_buff = nullptr) // used only with SMEM = false
+{
+   constexpr int DIM = 3;
+   static constexpr int GRID = SMEM ? 0 : 128;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto B = Reshape(b, Q1D, D1D);
+   const auto G = Reshape(g, Q1D, D1D);
+   const auto X = Reshape(x, D1D, D1D, D1D, DIM, NE);
+   auto Y = Reshape(y, Q1D, Q1D, Q1D, NE);
+
+   real_t *GM = nullptr;
+   if (!SMEM)
+   {
+      const DeviceDofQuadLimits &limits = DeviceDofQuadLimits::Get();
+      const int max_q1d = T_Q1D ? T_Q1D : limits.MAX_Q1D;
+      const int max_d1d = T_D1D ? T_D1D : limits.MAX_D1D;
+      const int max_qd = std::max(max_q1d, max_d1d);
+      const int mem_size = max_qd * max_qd * max_qd * 9;
+      d_buff->SetSize(2*mem_size*GRID);
+      GM = d_buff->Write();
+   }
+
+   mfem::forall_3D_grid(NE, Q1D, Q1D, Q1D, GRID, [=] MFEM_HOST_DEVICE (int e)
+   {
+      static constexpr int MQ1 = T_Q1D ? T_Q1D :
+                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_Q1D);
+      static constexpr int MD1 = T_D1D ? T_D1D :
+                                 (SMEM ? DofQuadLimits::MAX_DET_1D : DofQuadLimits::MAX_D1D);
+      static constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
+      static constexpr int MSZ = MDQ * MDQ * MDQ * 9;
+
+      const int bid = MFEM_BLOCK_ID(x);
+      MFEM_SHARED real_t BG[2][MQ1*MD1];
+      MFEM_SHARED real_t SM0[SMEM?MSZ:1];
+      MFEM_SHARED real_t SM1[SMEM?MSZ:1];
+      real_t *lm0 = SMEM ? SM0 : GM + MSZ*bid;
+      real_t *lm1 = SMEM ? SM1 : GM + MSZ*(GRID+bid);
+      real_t (*DDD)[MD1*MD1*MD1] = (real_t (*)[MD1*MD1*MD1]) (lm0);
+      real_t (*DDQ)[MD1*MD1*MQ1] = (real_t (*)[MD1*MD1*MQ1]) (lm1);
+      real_t (*DQQ)[MD1*MQ1*MQ1] = (real_t (*)[MD1*MQ1*MQ1]) (lm0);
+      real_t (*QQQ)[MQ1*MQ1*MQ1] = (real_t (*)[MQ1*MQ1*MQ1]) (lm1);
+
+      kernels::internal::LoadX<MD1>(e,D1D,X,DDD);
+      kernels::internal::LoadBG<MD1,MQ1>(D1D,Q1D,B,G,BG);
+
+      kernels::internal::GradX<MD1,MQ1>(D1D,Q1D,BG,DDD,DDQ);
+      kernels::internal::GradY<MD1,MQ1>(D1D,Q1D,BG,DDQ,DQQ);
+      kernels::internal::GradZ<MD1,MQ1>(D1D,Q1D,BG,DQQ,QQQ);
+
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               real_t J[9];
+               kernels::internal::PullGrad<MQ1>(Q1D, qx,qy,qz, QQQ, J);
+               Y(qx,qy,qz,e) = kernels::Det<3>(J);
+            }
+         }
+      }
+   });
+}
+
+} // namespace quadrature_interpolator
+} // namespace internal
+
+/// @cond Suppress_Doxygen_warnings
+
+template<int DIM, int SDIM, int D1D, int Q1D>
+QuadratureInterpolator::DetKernelType
+QuadratureInterpolator::DetKernels::Kernel()
+{
+   if (DIM == 1) { return internal::quadrature_interpolator::Det1D; }
+   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D<D1D, Q1D>; }
+   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface<D1D, Q1D>; }
+   else if (DIM == 3) { return internal::quadrature_interpolator::Det3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
+}
+
+/// @endcond
+
+} // namespace mfem
+
+#endif // MFEM_QUADINTERP_DET_HPP

fem/qspace.cpp

@@ -97,16 +97,31 @@ void QuadratureSpaceBase::Integrate(VectorCoefficient &coeff,
 void QuadratureSpace::ConstructOffsets()
 {
    const int num_elem = mesh.GetNE();
-   offsets.SetSize(num_elem + 1);
-   int offset = 0;
-   for (int i = 0; i < num_elem; i++)
+   ne = num_elem;
+
+   if (mesh.GetNumGeometries(mesh.Dimension()) == 1)
    {
-      offsets[i] = offset;
-      int geom = mesh.GetElementBaseGeometry(i);
-      MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule.");
-      offset += int_rule[geom]->GetNPoints();
+      Array<Geometry::Type> geoms;
+      mesh.GetGeometries(mesh.Dimension(), geoms);
+      offsets.SetSize(1);
+      offsets.HostWrite();
+      offsets[0] = int_rule[geoms[0]]->GetNPoints();
+      size = num_elem * offsets[0];
+   }
+   else
+   {
+      offsets.SetSize(num_elem + 1);
+      int offset = 0;
+      for (int i = 0; i < num_elem; i++)
+      {
+         offsets[i] = offset;
+         const Geometry::Type geom = mesh.GetElementBaseGeometry(i);
+         MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule.");
+         offset += int_rule[geom]->GetNPoints();
+      }
+      offsets[num_elem] = offset;
+      size = offsets.Last();
    }
-   offsets[num_elem] = size = offset;
 }
 
 void QuadratureSpace::Construct()
@@ -188,6 +203,7 @@ void FaceQuadratureSpace::ConstructOffsets()
 {
    face_indices.SetSize(num_faces);
    offsets.SetSize(num_faces + 1);
+   ne = num_faces;
    int offset = 0;
    int f_idx = 0;
    for (int i = 0; i < mesh.GetNumFacesWithGhost(); i++)

fem/qspace.hpp

@@ -30,13 +30,17 @@ protected:
    Mesh &mesh; ///< The underlying mesh.
    int order; ///< The order of integration rule.
    int size; ///< Total number of quadrature points.
+   int ne; ///< Actual number of entities
    mutable Vector weights; ///< Integration weights.
    mutable long nodes_sequence = 0; ///< Nodes counter for cache invalidation.
 
-   /// @brief Entity quadrature point offset array, of size num_entities + 1.
+   /// @brief Entity quadrature point offset array.
+   ///
+   /// Supports a constant compression scheme for meshes which have a single
+   /// geometry type. When compressed, will have a single value. The true offset
+   /// can be computed as i * offsets[0], where i is the entity index. Otherwise
+   /// has size num_entities + 1.
    ///
-   /// The quadrature point values for entity i are stored in the indices between
-   /// offsets[i] and offsets[i+1].
    Array<int> offsets;
    /// The quadrature rules used for each geometry type.
    const IntegrationRule *int_rule[Geometry::NumGeom];
@@ -59,6 +63,24 @@ protected:
    void ConstructWeights() const;
 
 public:
+   /// @brief Gets the offset for a given entity @a idx.
+   ///
+   /// The quadrature point values for entity i are stored in the indices
+   /// between Offset(i) and Offset(i+1)
+   int Offset(int idx) const
+   {
+      return (offsets.Size() == 1) ? (idx * offsets[0]) : offsets[idx];
+   }
+
+   /// @brief Entity quadrature point offset array.
+   ///
+   /// Supports a constant compression scheme for meshes which have a single
+   /// geometry type. When compressed, will have a single value. The true offset
+   /// can be computed as i * offsets[0], where i is the entity index. Otherwise
+   /// has size num_entities + 1.
+   ///
+   const Array<int> &Offsets() const { return offsets; }
+
    /// Return the total number of quadrature points.
    int GetSize() const { return size; }
 
@@ -66,7 +88,7 @@ public:
    int GetOrder() const { return order; }
 
    /// Return the number of entities.
-   int GetNE() const { return offsets.Size() - 1; }
+   int GetNE() const { return ne; }
 
    /// Returns the mesh.
    inline Mesh *GetMesh() const { return &mesh; }

fem/tmop.cpp

@@ -5122,33 +5122,32 @@ real_t TMOP_Integrator::GetSurfaceFittingWeight()
 
 void TMOP_Integrator::EnableNormalization(const GridFunction &x)
 {
-   ComputeNormalizationEnergies(x, metric_normal, lim_normal, surf_fit_normal);
+   ComputeNormalizationEnergies(x, metric_normal, lim_normal);
    metric_normal = 1.0 / metric_normal;
    lim_normal = 1.0 / lim_normal;
-   //if (surf_fit_gf) { surf_fit_normal = 1.0 / surf_fit_normal; }
    if (surf_fit_gf || surf_fit_pos) { surf_fit_normal = lim_normal; }
 }
 
 #ifdef MFEM_USE_MPI
 void TMOP_Integrator::ParEnableNormalization(const ParGridFunction &x)
 {
-   real_t loc[3];
-   ComputeNormalizationEnergies(x, loc[0], loc[1], loc[2]);
-   real_t rdc[3];
-   MPI_Allreduce(loc, rdc, 3, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+   real_t loc[2];
+   ComputeNormalizationEnergies(x, loc[0], loc[1]);
+   real_t rdc[2];
+   MPI_Allreduce(loc, rdc, 2, MPITypeMap<real_t>::mpi_type, MPI_SUM,
                  x.ParFESpace()->GetComm());
    metric_normal = 1.0 / rdc[0];
    lim_normal    = 1.0 / rdc[1];
-   // if (surf_fit_gf) { surf_fit_normal = 1.0 / rdc[2]; }
    if (surf_fit_gf || surf_fit_pos) { surf_fit_normal = lim_normal; }
 }
 #endif
 
 void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
                                                    real_t &metric_energy,
-                                                   real_t &lim_energy,
-                                                   real_t &surf_fit_gf_energy)
+                                                   real_t &lim_energy)
 {
+   metric_energy = 0.0;
+   lim_energy = 0.0;
    if (PA.enabled)
    {
       MFEM_VERIFY(PA.E.Size() > 0, "Must be called after AssemblePA!");
@@ -5191,9 +5190,6 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
    Jpr.SetSize(dim);
    Jpt.SetSize(dim);
 
-   metric_energy = 0.0;
-   lim_energy = 0.0;
-   surf_fit_gf_energy = 0.0;
    for (int i = 0; i < fes->GetNE(); i++)
    {
       const FiniteElement *fe = fes->GetFE(i);
@@ -5225,21 +5221,7 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
          lim_energy += weight;
       }
 
-      // Normalization of the surface fitting term.
-      if (surf_fit_gf)
-      {
-         Array<int> dofs;
-         Vector sigma_e;
-         surf_fit_gf->FESpace()->GetElementDofs(i, dofs);
-         surf_fit_gf->GetSubVector(dofs, sigma_e);
-         for (int s = 0; s < dofs.Size(); s++)
-         {
-            if ((*surf_fit_marker)[dofs[s]] == true)
-            {
-               surf_fit_gf_energy += sigma_e(s) * sigma_e(s);
-            }
-         }
-      }
+      // TODO: Normalization of the surface fitting term.
    }
 
    // Cases when integration is not over the target element, or when the

fem/tmop.hpp

@@ -2038,8 +2038,7 @@ protected:
    } PA;
 
    void ComputeNormalizationEnergies(const GridFunction &x,
-                                     real_t &metric_energy, real_t &lim_energy,
-                                     real_t &surf_fit_gf_energy);
+                                     real_t &metric_energy, real_t &lim_energy);
 
    void AssembleElementVectorExact(const FiniteElement &el,
                                    ElementTransformation &T,

general/array.hpp

@@ -326,7 +326,11 @@ public:
        the Size to match this Capacity after this.*/
    template <typename U>
    inline void CopyFrom(const U *src)
-   { std::memcpy(begin(), src, MemoryUsage()); }
+   {
+      if (!begin() || size == 0) { return; }
+      MFEM_ASSERT(begin() && src, "Error in Array::CopyFrom");
+      std::memcpy(begin(), src, MemoryUsage());
+   }
 
    /// STL-like begin.  Returns pointer to the first element of the array.
    inline T* begin() { return data; }

general/backends.hpp

@@ -14,7 +14,7 @@
 
 #include "../config/config.hpp"
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 #include <cusparse.h>
 #include <library_types.h>
 #include <cuda_runtime.h>
@@ -22,7 +22,7 @@
 #endif
 #include "cuda.hpp"
 
-#ifdef MFEM_USE_HIP
+#if defined(MFEM_USE_HIP) && defined(__HIP__)
 #include <hip/hip_runtime.h>
 #endif
 #include "hip.hpp"
@@ -43,7 +43,7 @@
 #endif
 #endif
 
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
+#if !defined(MFEM_USE_CUDA_OR_HIP)
 #define MFEM_DEVICE
 #define MFEM_HOST
 #define MFEM_LAMBDA
@@ -55,17 +55,18 @@
 #endif
 
 #if !((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \
-      (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
+      (defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__)))
 #define MFEM_SHARED
 #define MFEM_SYNC_THREAD
 #define MFEM_BLOCK_ID(k) 0
 #define MFEM_THREAD_ID(k) 0
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
+#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA
-#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)
 MFEM_DEVICE inline mfem::real_t atomicAdd(mfem::real_t *add, mfem::real_t val)
 {
    unsigned long long int *ptr = (unsigned long long int *) add;
@@ -93,7 +94,7 @@ template <typename T>
 MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 {
 #if ((defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)) || \
-     (defined(MFEM_USE_HIP)  && defined(__HIP_DEVICE_COMPILE__)))
+     (defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__)))
    return atomicAdd(&add,val);
 #else
    T old = add;

general/communication.cpp

@@ -347,13 +347,23 @@ void GroupTopology::Swap(GroupTopology &other)
    mfem::Swap(group_mgroup, other.group_mgroup);
 }
 
+/// \cond DO_NOT_DOCUMENT
 // Initialize the static mpi_type for the specializations of MPITypeMap:
+const MPI_Datatype MPITypeMap<bool>::mpi_type = MFEM_MPI_CXX_BOOL;
+const MPI_Datatype MPITypeMap<char>::mpi_type = MPI_CHAR;
+const MPI_Datatype MPITypeMap<unsigned char>::mpi_type = MPI_UNSIGNED_CHAR;
+const MPI_Datatype MPITypeMap<short>::mpi_type = MPI_SHORT;
+const MPI_Datatype MPITypeMap<unsigned short>::mpi_type = MPI_UNSIGNED_SHORT;
 const MPI_Datatype MPITypeMap<int>::mpi_type = MPI_INT;
+const MPI_Datatype MPITypeMap<unsigned int>::mpi_type = MPI_UNSIGNED;
+const MPI_Datatype MPITypeMap<long>::mpi_type = MPI_LONG;
+const MPI_Datatype MPITypeMap<unsigned long>::mpi_type = MPI_UNSIGNED_LONG;
+const MPI_Datatype MPITypeMap<long long>::mpi_type = MPI_LONG_LONG;
+const MPI_Datatype MPITypeMap<unsigned long long>::mpi_type =
+   MPI_UNSIGNED_LONG_LONG;
 const MPI_Datatype MPITypeMap<float>::mpi_type = MPI_FLOAT;
 const MPI_Datatype MPITypeMap<double>::mpi_type = MPI_DOUBLE;
-const MPI_Datatype MPITypeMap<int64_t>::mpi_type = MPI_INT64_T;
-const MPI_Datatype MPITypeMap<uint64_t>::mpi_type = MPI_UINT64_T;
-
+/// \endcond DO_NOT_DOCUMENT
 
 GroupCommunicator::GroupCommunicator(const GroupTopology &gt, Mode m)
    : gtopo(gt), mode(m)

general/communication.hpp

@@ -425,16 +425,24 @@ public:
    ~GroupCommunicator();
 };
 
+/// General MPI message tags used by MFEM
+enum MessageTag
+{
+   DEREFINEMENT_MATRIX_CONSTRUCTION_DATA =
+      291, /// ParFiniteElementSpace ParallelDerefinementMatrix and
+   /// ParDerefineMatrixOp
+};
+
 enum VarMessageTag
 {
-   NEIGHBOR_ELEMENT_RANK_VM,  ///< NeighborElementRankMessage
-   NEIGHBOR_ORDER_VM,         ///< NeighborOrderMessage
-   NEIGHBOR_DEREFINEMENT_VM,  ///< NeighborDerefinementMessage
-   NEIGHBOR_REFINEMENT_VM,    ///< NeighborRefinementMessage
-   NEIGHBOR_PREFINEMENT_VM,   ///< NeighborPRefinementMessage
-   NEIGHBOR_ROW_VM,           ///< NeighborRowMessage
-   REBALANCE_VM,              ///< RebalanceMessage
-   REBALANCE_DOF_VM           ///< RebalanceDofMessage
+   NEIGHBOR_ELEMENT_RANK_VM, ///< NeighborElementRankMessage
+   NEIGHBOR_ORDER_VM,        ///< NeighborOrderMessage
+   NEIGHBOR_DEREFINEMENT_VM, ///< NeighborDerefinementMessage
+   NEIGHBOR_REFINEMENT_VM,   ///< NeighborRefinementMessage
+   NEIGHBOR_PREFINEMENT_VM,  ///< NeighborPRefinementMessage
+   NEIGHBOR_ROW_VM,          ///< NeighborRowMessage
+   REBALANCE_VM,             ///< RebalanceMessage
+   REBALANCE_DOF_VM,         ///< RebalanceDofMessage
 };
 
 /// \brief Variable-length MPI message containing unspecific binary data.
@@ -599,10 +607,50 @@ protected:
 template <typename Type> struct MPITypeMap;
 
 // Specializations of MPITypeMap; mpi_type initialized in communication.cpp:
+template<> struct MPITypeMap<bool>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<char>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<unsigned char>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<short>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<unsigned short>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
 template<> struct MPITypeMap<int>
 {
    static MFEM_EXPORT const MPI_Datatype mpi_type;
 };
+template<> struct MPITypeMap<unsigned int>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<long>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<unsigned long>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<long long>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
+template<> struct MPITypeMap<unsigned long long>
+{
+   static MFEM_EXPORT const MPI_Datatype mpi_type;
+};
 template<> struct MPITypeMap<double>
 {
    static MFEM_EXPORT const MPI_Datatype mpi_type;
@@ -611,15 +659,6 @@ template<> struct MPITypeMap<float>
 {
    static MFEM_EXPORT const MPI_Datatype mpi_type;
 };
-template<> struct MPITypeMap<int64_t>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-template<> struct MPITypeMap<uint64_t>
-{
-   static MFEM_EXPORT const MPI_Datatype mpi_type;
-};
-
 
 /** Reorder MPI ranks to follow the Z-curve within the physical machine topology
     (provided that functions to query physical node coordinates are available).

general/cuda.cpp

@@ -24,7 +24,7 @@ void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                      const char *file, int line)
 {
    mfem::err << "\n\nCUDA error: (" << expr << ") failed with error:\n --> "
-             << cudaGetErrorString(err)
+             << cudaGetErrorString(err) << " [code: " << (int)err << ']'
              << "\n ... in function: " << func
              << "\n ... in file: " << file << ':' << line << '\n';
    mfem_error();

general/cuda.hpp

@@ -18,7 +18,7 @@
 // CUDA block size used by MFEM.
 #define MFEM_CUDA_BLOCKS 256
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 #define MFEM_USE_CUDA_OR_HIP
 #define MFEM_DEVICE __device__
 #define MFEM_HOST __host__
@@ -37,22 +37,23 @@
                               __FILE__, __LINE__);                             \
     }                                                                          \
   } while (0)
-#endif // MFEM_USE_CUDA
 
 // Define the MFEM inner threading macros
-#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)
+#if defined(__CUDA_ARCH__)
 #define MFEM_SHARED __shared__
 #define MFEM_SYNC_THREAD __syncthreads()
 #define MFEM_BLOCK_ID(k) blockIdx.k
 #define MFEM_THREAD_ID(k) threadIdx.k
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
-#endif
+#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
+#endif // defined(__CUDA_ARCH__)
+#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 
 namespace mfem
 {
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 // Function used by the macro MFEM_GPU_CHECK.
 void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
                      const char *file, int line);

general/device.cpp

@@ -16,6 +16,7 @@
 #include "../fem/ceed/interface/util.hpp"
 #endif
 #ifdef MFEM_USE_MPI
+#include "communication.hpp"
 #include "../linalg/hypre.hpp"
 #endif
 
@@ -145,6 +146,11 @@ Device::Device()
       Configure(device);
       device_env = true;
    }
+
+   if (GetEnv("MFEM_GPU_AWARE_MPI"))
+   {
+      SetGPUAwareMPI(true);
+   }
 }
 
 Device::~Device()
@@ -196,6 +202,29 @@ void Device::Configure(const std::string &device, const int device_id)
    {
       bmap[internal::backend_name[i]] = internal::backend_list[i];
    }
+   // auto-detect GPU configurations
+   // assumes only one of HIP or CUDA are available
+#ifdef MFEM_USE_HIP
+   bmap["gpu"] = Backend::HIP;
+#ifdef MFEM_USE_RAJA
+   bmap["raja-gpu"] = Backend::RAJA_HIP;
+#endif
+#ifdef MFEM_USE_CEED
+   bmap["ceed-gpu"] = Backend::CEED_HIP;
+#endif
+   // no OCCA+HIP?
+#elif defined(MFEM_USE_CUDA)
+   bmap["gpu"] = Backend::CUDA;
+#ifdef MFEM_USE_RAJA
+   bmap["raja-gpu"] = Backend::RAJA_CUDA;
+#endif
+#ifdef MFEM_USE_CEED
+   bmap["ceed-gpu"] = Backend::CEED_CUDA;
+#endif
+#ifdef MFEM_USE_OCCA
+   bmap["occa-gpu"] = Backend::OCCA_CUDA;
+#endif
+#endif
    std::string device_option;
    std::string::size_type beg = 0, end;
    while (1)
@@ -313,6 +342,13 @@ void Device::Print(std::ostream &os)
    {
       os << ',' << MemoryTypeName[static_cast<int>(device_mem_type)];
    }
+#ifdef MFEM_USE_MPI
+   if (Allows(Backend::DEVICE_MASK) &&
+       Mpi::IsInitialized() && !Mpi::IsFinalized())
+   {
+      os << "\nUse GPU-aware MPI:    " << (GetGPUAwareMPI() ? "yes" : "no");
+   }
+#endif
    os << std::endl;
 }
 
@@ -579,7 +615,7 @@ void Device::Setup(const std::string &device_option, const int device_id)
    if (Allows(Backend::DEBUG_DEVICE)) { ngpu = 1; }
 }
 
-MemoryType Device::QueryMemoryType(void *ptr)
+MemoryType Device::QueryMemoryType(const void* ptr)
 {
    // from HYPRE's hypre_GetPointerLocation
    MemoryType res = MemoryType::HOST;

general/device.hpp

@@ -198,6 +198,10 @@ public:
          'ceed-hip', 'hip', 'debug',
          'occa-omp', 'raja-omp', 'omp',
          'ceed-cpu', 'occa-cpu', 'raja-cpu', 'cpu'.
+       - The following backend aliases are also available: 'ceed-gpu',
+         'occa-gpu', 'raja-gpu', and 'gpu' where they alias their respective
+         '*-cuda' or '*-hip' backends depending on the MFEM build-time
+         configuration.
        - Multiple backends can be configured at the same time.
        - Only one 'occa-*' backend can be configured at a time.
        - The backend 'occa-cuda' enables the 'cuda' backend unless 'raja-cuda'
@@ -293,9 +297,9 @@ public:
    /// Get the status of GPU-aware MPI flag.
    static bool GetGPUAwareMPI() { return Get().mpi_gpu_aware; }
 
-   /** @brief Query the device driver for what memory type a given @a ptr is
-       allocated with. */
-   static MemoryType QueryMemoryType(void *ptr);
+   /** Query the device driver for what memory type a given @a ptr is allocated
+    * with. */
+   static MemoryType QueryMemoryType(const void* ptr);
 
    /** @brief The number of hardware compute units/streaming multiprocessors
        available on a given compute device @a device_id. */

general/error.hpp

@@ -176,7 +176,7 @@ __device__ void abort_msg(T & msg)
       printf(__VA_ARGS__);     \
       asm("trap;");            \
    }
-#elif defined(MFEM_USE_HIP)
+#elif defined(__HIP_DEVICE_COMPILE__)
 #define MFEM_ABORT_KERNEL(...) \
    {                           \
       printf(__VA_ARGS__);     \

general/forall.hpp

@@ -158,8 +158,8 @@ private:
 #define MFEM_PRAGMA(X) _Pragma(#X)
 
 // MFEM_UNROLL pragma macro that can be used inside MFEM_FORALL macros.
-#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)
-#ifdef __NVCC__
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) // Clang cuda or nvcc
+#ifdef __NVCC__ // nvcc specifically
 #define MFEM_UNROLL(N) MFEM_PRAGMA(unroll(N))
 #else // Assuming Clang CUDA
 #define MFEM_UNROLL(N) MFEM_PRAGMA(unroll N)
@@ -169,12 +169,12 @@ private:
 #endif
 
 // MFEM_GPU_FORALL: "parallel for" executed with CUDA or HIP based on the MFEM
-// build-time configuration (MFEM_USE_CUDA or MFEM_USE_HIP). If neither CUDA nor
-// HIP is enabled, this macro is a no-op.
-#if defined(MFEM_USE_CUDA)
+// build-time configuration (MFEM_USE_CUDA or MFEM_USE_HIP), and if compiling
+// with CUDA/HIP language. Otherwise, this macro is a no-op.
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 #define MFEM_GPU_FORALL(i, N,...) CuWrap1D(N, [=] MFEM_DEVICE      \
                                        (int i) {__VA_ARGS__})
-#elif defined(MFEM_USE_HIP)
+#elif defined(MFEM_USE_HIP) && defined(__HIP__)
 #define MFEM_GPU_FORALL(i, N,...) HipWrap1D(N, [=] MFEM_DEVICE     \
                                         (int i) {__VA_ARGS__})
 #else
@@ -225,6 +225,52 @@ void OmpWrap(const int N, HBODY &&h_body)
 #endif
 }
 
+template <typename HBODY>
+void OmpWrap2D(const int Nx, const int Ny, HBODY &&h_body)
+{
+#ifdef MFEM_USE_OPENMP
+   // requires OpenMP 3.1
+   #pragma omp parallel for collapse(2)
+   for (int j = 0; j < Ny; j++)
+   {
+      for (int i = 0; i < Nx; i++)
+      {
+         h_body(i, j);
+      }
+   }
+#else
+   MFEM_CONTRACT_VAR(Nx);
+   MFEM_CONTRACT_VAR(Ny);
+   MFEM_CONTRACT_VAR(h_body);
+   MFEM_ABORT("OpenMP requested for MFEM but OpenMP is not enabled!");
+#endif
+}
+
+template <typename HBODY>
+void OmpWrap3D(const int Nx, const int Ny, const int Nz, HBODY &&h_body)
+{
+#ifdef MFEM_USE_OPENMP
+   // requires OpenMP 3.1
+   #pragma omp parallel for collapse(3)
+   for (int k = 0; k < Nz; k++)
+   {
+      for (int j = 0; j < Ny; j++)
+      {
+         for (int i = 0; i < Nx; i++)
+         {
+            h_body(i, j, k);
+         }
+      }
+   }
+#else
+   MFEM_CONTRACT_VAR(Nx);
+   MFEM_CONTRACT_VAR(Ny);
+   MFEM_CONTRACT_VAR(Nz);
+   MFEM_CONTRACT_VAR(h_body);
+   MFEM_ABORT("OpenMP requested for MFEM but OpenMP is not enabled!");
+#endif
+}
+
 
 /// RAJA Cuda and Hip backends
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA)
@@ -454,6 +500,42 @@ void RajaOmpWrap(const int N, HBODY &&h_body)
    RAJA::forall<RAJA::omp_parallel_for_exec>(RAJA::RangeSegment(0,N), h_body);
 }
 
+template <typename HBODY>
+void RajaOmpWrap2D(const int Nx, const int Ny, HBODY &&h_body)
+{
+   using omp_launch_policy = RAJA::LaunchPolicy<RAJA::omp_launch_t>;
+   using global_thread_xy = RAJA::LoopPolicy<RAJA::omp_for_exec>;
+   RAJA::RangeSegment xrange(0, Nx);
+   RAJA::RangeSegment yrange(0, Ny);
+   RAJA::launch<omp_launch_policy>(RAJA::ExecPlace::HOST, RAJA::LaunchParams(),
+                                   [=](RAJA::LaunchContext ctx)
+   {
+      // contiguous in x
+      RAJA::expt::loop<global_thread_xy>(ctx, xrange, yrange, [&](int i, int j)
+      {
+         h_body(i, j);
+      });
+   });
+}
+
+template <typename HBODY>
+void RajaOmpWrap3D(const int Nx, const int Ny, const int Nz, HBODY &&h_body)
+{
+   using omp_launch_policy = RAJA::LaunchPolicy<RAJA::omp_launch_t>;
+   using global_thread_xyz = RAJA::LoopPolicy<RAJA::omp_for_exec>;
+   RAJA::RangeSegment xrange(0, Nx);
+   RAJA::RangeSegment yrange(0, Ny);
+   RAJA::RangeSegment zrange(0, Nz);
+   RAJA::launch<omp_launch_policy>(RAJA::ExecPlace::HOST, RAJA::LaunchParams(),
+                                   [=](RAJA::LaunchContext ctx)
+   {
+      // contiguous in x
+      RAJA::expt::loop<global_thread_xyz>(ctx, xrange, yrange, zrange,
+                                          [&](int i, int j, int k)
+      { h_body(i, j, k); });
+   });
+}
+
 #endif
 
 
@@ -481,7 +563,7 @@ void RajaSeqWrap(const int N, HBODY &&h_body)
 
 
 /// CUDA backend
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 
 template <typename BODY> __global__ static
 void CuKernel1D(const int N, BODY body)
@@ -573,11 +655,11 @@ struct CuWrap<3>
    }
 };
 
-#endif // MFEM_USE_CUDA
+#endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 
 
 /// HIP backend
-#ifdef MFEM_USE_HIP
+#if defined(MFEM_USE_HIP) && defined(__HIP__)
 
 template <typename BODY> __global__ static
 void HipKernel1D(const int N, BODY body)
@@ -668,7 +750,7 @@ struct HipWrap<3>
    }
 };
 
-#endif // MFEM_USE_HIP
+#endif // defined(MFEM_USE_HIP) && defined(__HIP__)
 
 
 /// The forall kernel body wrapper
@@ -701,7 +783,7 @@ inline void ForallWrap(const bool use_dev, const int N,
    }
 #endif
 
-#ifdef MFEM_USE_CUDA
+#if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
    // If Backend::CUDA is allowed, use it
    if (Device::Allows(Backend::CUDA))
    {
@@ -709,7 +791,7 @@ inline void ForallWrap(const bool use_dev, const int N,
    }
 #endif
 
-#ifdef MFEM_USE_HIP
+#if defined(MFEM_USE_HIP) && defined(__HIP__)
    // If Backend::HIP is allowed, use it
    if (Device::Allows(Backend::HIP))
    {
@@ -753,6 +835,83 @@ inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
 template<typename lambda>
 inline void forall(int N, lambda &&body) { ForallWrap<1>(true, N, body); }
 
+template<typename lambda>
+inline void forall(int Nx, int Ny, lambda &&body)
+{
+   if (Device::Allows(Backend::DEVICE_MASK))
+   {
+      forall(Nx * Ny, [=] MFEM_HOST_DEVICE(int idx)
+      {
+         int j = idx / Nx;
+         int i = idx % Nx;
+         body(i, j);
+      });
+   }
+#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
+   else if (Device::Allows(Backend::RAJA_OMP))
+   {
+      return RajaOmpWrap2D(Nx, Ny, body);
+   }
+#endif
+#ifdef MFEM_USE_OPENMP
+   else if (Device::Allows(Backend::OMP))
+   {
+      return OmpWrap2D(Nx, Ny, body);
+   }
+#endif
+   else
+   {
+      for (int j = 0; j < Ny; ++j)
+      {
+         for (int i = 0; i < Nx; ++i)
+         {
+            body(i, j);
+         }
+      }
+   }
+}
+
+template<typename lambda>
+inline void forall(int Nx, int Ny, int Nz, lambda &&body)
+{
+   if (Device::Allows(Backend::DEVICE_MASK))
+   {
+      forall(Nx * Ny * Nz, [=] MFEM_HOST_DEVICE(int idx)
+      {
+         int i = idx % Nx;
+         int j = idx / Nx;
+         int k = j / Ny;
+         j = j % Ny;
+         body(i, j, k);
+      });
+   }
+#if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
+   else if (Device::Allows(Backend::RAJA_OMP))
+   {
+      return RajaOmpWrap3D(Nx, Ny, Nz, body);
+   }
+#endif
+#ifdef MFEM_USE_OPENMP
+   else if (Device::Allows(Backend::OMP))
+   {
+      return OmpWrap3D(Nx, Ny, Nz, body);
+   }
+#endif
+   else
+   {
+      for (int k = 0; k < Nz; ++k)
+      {
+         for (int j = 0; j < Ny; ++j)
+         {
+            for (int i = 0; i < Nx; ++i)
+            {
+               body(i, j, k);
+            }
+         }
+      }
+   }
+}
+
 template<typename lambda>
 inline void forall_switch(bool use_dev, int N, lambda &&body)
 {

general/hip.cpp

@@ -24,7 +24,7 @@ void mfem_hip_error(hipError_t err, const char *expr, const char *func,
                     const char *file, int line)
 {
    mfem::err << "\n\nHIP error: (" << expr << ") failed with error:\n --> "
-             << hipGetErrorString(err)
+             << hipGetErrorString(err) << " [code: " << (int)err << ']'
              << "\n ... in function: " << func
              << "\n ... in file: " << file << ':' << line << '\n';
    mfem_error();

general/hip.hpp

@@ -18,7 +18,7 @@
 // HIP block size used by MFEM.
 #define MFEM_HIP_BLOCKS 256
 
-#ifdef MFEM_USE_HIP
+#if defined(MFEM_USE_HIP) && defined(__HIP__)
 #define MFEM_USE_CUDA_OR_HIP
 #define MFEM_DEVICE __device__
 #define MFEM_HOST __host__
@@ -37,18 +37,20 @@
                              __FILE__, __LINE__);                              \
     }                                                                          \
   } while (0)
-#endif // MFEM_USE_HIP
 
 // Define the MFEM inner threading macros
-#if defined(MFEM_USE_HIP) && defined(__HIP_DEVICE_COMPILE__)
+#if defined(__HIP_DEVICE_COMPILE__)
 #define MFEM_SHARED __shared__
 #define MFEM_SYNC_THREAD __syncthreads()
 #define MFEM_BLOCK_ID(k) hipBlockIdx_ ##k
 #define MFEM_THREAD_ID(k) hipThreadIdx_ ##k
 #define MFEM_THREAD_SIZE(k) hipBlockDim_ ##k
 #define MFEM_FOREACH_THREAD(i,k,N) \
-    for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
-#endif
+   for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
+#define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
+   if(const int i=hipThreadIdx_ ##k; i<N)
+#endif // defined(__HIP_DEVICE_COMPILE__)
+#endif // defined(MFEM_USE_HIP) && defined(__HIP__)
 
 namespace mfem
 {

general/mem_manager.cpp

@@ -513,10 +513,7 @@ public:
    void *HtoD(void *dst, const void *src, size_t bytes) override
    { return HipMemcpyHtoD(dst, src, bytes); }
    void *DtoD(void* dst, const void* src, size_t bytes) override
-   // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so
-   // instead we use hipMemcpyAsync to get similar behavior.
-   // for more info see: https://github.com/mfem/mfem/pull/2780
-   { return HipMemcpyDtoDAsync(dst, src, bytes); }
+   { return HipMemcpyDtoD(dst, src, bytes); }
    void *DtoH(void *dst, const void *src, size_t bytes) override
    { return HipMemcpyDtoH(dst, src, bytes); }
 };
@@ -658,10 +655,7 @@ public:
       return CuMemcpyDtoD(dst, src, bytes);
 #endif
 #ifdef MFEM_USE_HIP
-      // Unlike cudaMemcpy(DtoD), hipMemcpy(DtoD) causes a host-side synchronization so
-      // instead we use hipMemcpyAsync to get similar behavior.
-      // for more info see: https://github.com/mfem/mfem/pull/2780
-      return HipMemcpyDtoDAsync(dst, src, bytes);
+      return HipMemcpyDtoD(dst, src, bytes);
 #endif
       // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }

general/mem_manager.hpp

@@ -896,6 +896,7 @@ inline HYPRE_MemoryLocation GetHypreMemoryLocation()
 #elif MFEM_HYPRE_VERSION < 23100
    return HYPRE_MEMORY_DEVICE;
 #else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
+   if (!HYPRE_Initialized()) { return HYPRE_MEMORY_HOST; }
    HYPRE_MemoryLocation loc;
    HYPRE_GetMemoryLocation(&loc);
    return loc;
@@ -1057,7 +1058,8 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
          // register the 'base' if the MemoryManager::Exists():
          MemoryManager::Exists()
 #else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-         MemoryManager::Exists() && HypreUsingGPU()
+         IsDeviceMemory(MemoryManager::GetDeviceMemoryType()) ||
+         (MemoryManager::Exists() && HypreUsingGPU())
 #endif
       )
       {

general/reducers.hpp

@@ -537,7 +537,7 @@ void reduce(int N, T &res, B &&body, const R &reducer, bool use_dev,
       return;
    }
 
-#if defined(MFEM_USE_HIP) || defined(MFEM_USE_CUDA)
+#if defined(MFEM_USE_CUDA_OR_HIP)
    if (use_dev &&
        mfem::Device::Allows(Backend::CUDA | Backend::HIP | Backend::RAJA_CUDA |
                             Backend::RAJA_HIP))

linalg/densemat.cpp

@@ -4405,4 +4405,32 @@ void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X)
    BatchedLinAlg::LUSolve(Mlu, P, X);
 }
 
+#ifdef MFEM_USE_LAPACK
+void BandedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
+                 Array<int> &ipiv)
+{
+   int LDAB = (2*KL) + KU + 1;
+   int N = AB.NumCols();
+   int NRHS = B.NumCols();
+   int info;
+   ipiv.SetSize(N);
+   MFEM_LAPACK_PREFIX(gbsv_)(&N, &KL, &KU, &NRHS, AB.GetData(), &LDAB,
+                             ipiv.GetData(), B.GetData(), &N, &info);
+   MFEM_ASSERT(info == 0, "BandedSolve failed in LAPACK");
+}
+
+void BandedFactorizedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
+                           bool transpose, Array<int> &ipiv)
+{
+   int LDAB = (2*KL) + KU + 1;
+   int N = AB.NumCols();
+   int NRHS = B.NumCols();
+   char trans = transpose ? 'T' : 'N';
+   int info;
+   MFEM_LAPACK_PREFIX(gbtrs_)(&trans, &N, &KL, &KU, &NRHS, AB.GetData(), &LDAB,
+                              ipiv.GetData(), B.GetData(), &N, &info);
+   MFEM_ASSERT(info == 0, "BandedFactorizedSolve failed in LAPACK");
+}
+#endif
+
 } // namespace mfem

linalg/densemat.hpp

@@ -1329,6 +1329,13 @@ void BatchLUFactor(DenseTensor &Mlu, Array<int> &P, const real_t TOL = 0.0);
     dimension m x n. */
 void BatchLUSolve(const DenseTensor &Mlu, const Array<int> &P, Vector &X);
 
+#ifdef MFEM_USE_LAPACK
+void BandedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
+                 Array<int> &ipiv);
+void BandedFactorizedSolve(int KL, int KU, DenseMatrix &AB, DenseMatrix &B,
+                           bool transpose, Array<int> &ipiv);
+#endif
+
 // Inline methods
 
 inline real_t &DenseMatrix::operator()(int i, int j)

linalg/hypre.cpp

@@ -2574,6 +2574,18 @@ void HypreParMatrix::EliminateBC(const Array<int> &ess_dofs,
 #if defined(HYPRE_USING_GPU)
       if (HypreUsingGPU())
       {
+#if defined(HYPRE_WITH_GPU_AWARE_MPI) || defined(HYPRE_USING_GPU_AWARE_MPI)
+         // hypre_GetGpuAwareMPI() was introduced in v2.31.0, however, its value
+         // is not checked in hypre_ParCSRCommHandleCreate_v2() before v2.33.0,
+         // instead only HYPRE_WITH_GPU_AWARE_MPI is checked.
+#if MFEM_HYPRE_VERSION >= 23300
+         if (hypre_GetGpuAwareMPI())
+#endif
+         {
+            // ensure int_buf_data has been computed before sending it
+            MFEM_STREAM_SYNC;
+         }
+#endif
          // Try to use device-aware MPI for the communication if available
          comm_handle = hypre_ParCSRCommHandleCreate_v2(
                           11, comm_pkg, HYPRE_MEMORY_DEVICE, int_buf_data,

linalg/lapack.hpp

@@ -42,6 +42,13 @@ extern "C" void
 MFEM_LAPACK_PREFIX(getri_)(int *N, real_t *A, int *LDA, int *IPIV, real_t *WORK,
                            int *LWORK, int *INFO);
 extern "C" void
+MFEM_LAPACK_PREFIX(gbsv_)(int *, int *, int *, int *, real_t *, int *, int *,
+                          real_t *, int *, int *);
+extern "C" void
+MFEM_LAPACK_PREFIX(gbtrs_)(char *, int *, int *, int *, int *, real_t *, int *,
+                           int *, real_t *, int *, int *);
+
+extern "C" void
 MFEM_LAPACK_PREFIX(syevr_)(char *JOBZ, char *RANGE, char *UPLO, int *N,
                            real_t *A, int *LDA, real_t *VL, real_t *VU, int *IL,
                            int *IU, real_t *ABSTOL, int *M, real_t *W,

linalg/vector.cpp

@@ -117,7 +117,7 @@ Vector::Vector(const Vector &v)
    UseDevice(v.UseDevice());
 }
 
-Vector::Vector(Vector &&v)
+Vector::Vector(Vector &&v) : Vector()
 {
    *this = std::move(v);
 }

mesh/exodus_writer.cpp

@@ -26,6 +26,12 @@
    }\
 }
 
+#if defined(MFEM_USE_DOUBLE)
+#define MFEM_NETCDF_REAL_T NC_DOUBLE
+#elif defined(MFEM_USE_SINGLE)
+#define MFEM_NETCDF_REAL_T NC_FLOAT
+#endif
+
 namespace mfem
 {
 
@@ -135,18 +141,18 @@ public:
    /// @brief Writes the mesh to an ExodusII file.
    /// @param fpath The path to the file.
    /// @param flags NC_CLOBBER will overwrite existing file.
-   void PrintExodusII(std::string fpath, int flags = NC_CLOBBER);
+   void PrintExodusII(const std::string &fpath, int flags = NC_CLOBBER);
 
    /// @brief Static method for writing a mesh to an ExodusII file.
    /// @param mesh The mesh to write to the file.
    /// @param fpath The path to the file.
    /// @param flags NetCDF file flags.
-   static void PrintExodusII(Mesh & mesh, std::string fpath,
+   static void PrintExodusII(Mesh & mesh, const std::string &fpath,
                              int flags = NC_CLOBBER);
 
 protected:
    /// @brief Closes any open file and creates a NetCDF file using selected flags.
-   void OpenExodusII(std::string fpath, int flags);
+   void OpenExodusII(const std::string &fpath, int flags);
 
    /// @brief Closes any open file.
    void CloseExodusII();
@@ -167,9 +173,9 @@ protected:
    std::unordered_set<int> GenerateUniqueNodeIDs();
 
    /// @brief Populates vectors with x, y, z coordinates from mesh.
-   void ExtractVertexCoordinates(std::vector<double> & coordx,
-                                 std::vector<double> & coordy,
-                                 std::vector<double> & coordz);
+   void ExtractVertexCoordinates(std::vector<real_t> &coordx,
+                                 std::vector<real_t> &coordy,
+                                 std::vector<real_t> &coordz);
 
    /// @brief Writes node connectivity for a particular block.
    /// @param block_id The block to write to the file.
@@ -187,7 +193,7 @@ protected:
    /// @brief Writes the number of elements in the mesh.
    void WriteNumOfElements();
 
-   /// @brief Writes the floating-point word size (4 == float; 8 == double).
+   /// @brief Writes the floating-point word size (sizeof(real_t)).
    void WriteFloatingPointWordSize();
 
    /// @brief Writes the API version.
@@ -291,7 +297,7 @@ private:
    std::map<int, std::vector<int>> exodusII_side_ids_for_boundary_id;
 };
 
-void Mesh::PrintExodusII(const std::string fpath)
+void Mesh::PrintExodusII(const std::string &fpath)
 {
    ExodusIIWriter::PrintExodusII(*this, fpath);
 }
@@ -362,7 +368,7 @@ void ExodusIIWriter::WriteExodusIIMeshInformation()
    WriteNodeSets();
 }
 
-void ExodusIIWriter::PrintExodusII(std::string fpath, int flags)
+void ExodusIIWriter::PrintExodusII(const std::string &fpath, int flags)
 {
    OpenExodusII(fpath, flags);
 
@@ -374,7 +380,7 @@ void ExodusIIWriter::PrintExodusII(std::string fpath, int flags)
    mfem::out << "Mesh successfully written to Exodus II file" << std::endl;
 }
 
-void ExodusIIWriter::PrintExodusII(Mesh & mesh, std::string fpath,
+void ExodusIIWriter::PrintExodusII(Mesh &mesh, const std::string &fpath,
                                    int flags)
 {
    ExodusIIWriter writer(mesh);
@@ -382,7 +388,7 @@ void ExodusIIWriter::PrintExodusII(Mesh & mesh, std::string fpath,
    writer.PrintExodusII(fpath, flags);
 }
 
-void ExodusIIWriter::OpenExodusII(std::string fpath, int flags)
+void ExodusIIWriter::OpenExodusII(const std::string &fpath, int flags)
 {
    CloseExodusII();  // Close any open files.
 
@@ -422,7 +428,7 @@ void ExodusIIWriter::WriteNumOfElements()
 
 void ExodusIIWriter::WriteFloatingPointWordSize()
 {
-   const int word_size = 8;
+   const int word_size = sizeof(real_t);
    PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_FLOATING_POINT_WORD_SIZE_LABEL,
           NC_INT, 1,
           &word_size);
@@ -430,13 +436,15 @@ void ExodusIIWriter::WriteFloatingPointWordSize()
 
 void ExodusIIWriter::WriteAPIVersion()
 {
-   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_API_VERSION_LABEL, NC_FLOAT, 1,
+   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_API_VERSION_LABEL, MFEM_NETCDF_REAL_T,
+          1,
           &ExodusIILabels::EXODUS_API_VERSION);
 }
 
 void ExodusIIWriter::WriteDatabaseVersion()
 {
-   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_DATABASE_VERSION_LABEL, NC_FLOAT, 1,
+   PutAtt(NC_GLOBAL, ExodusIILabels::EXODUS_DATABASE_VERSION_LABEL,
+          MFEM_NETCDF_REAL_T, 1,
           &ExodusIILabels::EXODUS_DATABASE_VERSION);
 }
 
@@ -607,25 +615,25 @@ void ExodusIIWriter::WriteNodalCoordinates()
    DefineDimension("num_nodes", num_nodes, &num_nodes_id);
 
    // 3. Extract the nodal coordinates.
-   // NB: assume doubles (could be floats!); ndims = 1 (vector).
+   // NB: writes in format real_t (double or float); ndims = 1 (vector).
    // https://docs.unidata.ucar.edu/netcdf-c/current/group__variables.html#gac7e8662c51f3bb07d1fc6d6c6d9052c8
-   std::vector<double> coordx(num_nodes);
-   std::vector<double> coordy(num_nodes);
-   std::vector<double> coordz(mesh.Dimension() == 3 ? num_nodes : 0);
+   std::vector<real_t> coordx(num_nodes);
+   std::vector<real_t> coordy(num_nodes);
+   std::vector<real_t> coordz(mesh.Dimension() == 3 ? num_nodes : 0);
 
    ExtractVertexCoordinates(coordx, coordy, coordz);
 
    // 4. Define and put the nodal coordinates.
-   DefineAndPutVar(ExodusIILabels::EXODUS_COORDX_LABEL, NC_DOUBLE, 1,
+   DefineAndPutVar(ExodusIILabels::EXODUS_COORDX_LABEL, MFEM_NETCDF_REAL_T, 1,
                    &num_nodes_id,
                    coordx.data());
-   DefineAndPutVar(ExodusIILabels::EXODUS_COORDY_LABEL, NC_DOUBLE, 1,
+   DefineAndPutVar(ExodusIILabels::EXODUS_COORDY_LABEL, MFEM_NETCDF_REAL_T, 1,
                    &num_nodes_id,
                    coordy.data());
 
    if (mesh.Dimension() == 3)
    {
-      DefineAndPutVar(ExodusIILabels::EXODUS_COORDZ_LABEL, NC_DOUBLE, 1,
+      DefineAndPutVar(ExodusIILabels::EXODUS_COORDZ_LABEL, MFEM_NETCDF_REAL_T, 1,
                       &num_nodes_id,
                       coordz.data());
    }
@@ -770,9 +778,9 @@ void ExodusIIWriter::WriteNodeConnectivityForBlock(const int block_id)
 }
 
 
-void ExodusIIWriter::ExtractVertexCoordinates(std::vector<double> & coordx,
-                                              std::vector<double> & coordy,
-                                              std::vector<double> & coordz)
+void ExodusIIWriter::ExtractVertexCoordinates(std::vector<real_t> & coordx,
+                                              std::vector<real_t> & coordy,
+                                              std::vector<real_t> & coordz)
 {
    if (mesh.GetNodes()) // Higher-order.
    {
@@ -782,7 +790,7 @@ void ExodusIIWriter::ExtractVertexCoordinates(std::vector<double> & coordx,
       sorted_node_ids.assign(unordered_node_ids.begin(), unordered_node_ids.end());
       std::sort(sorted_node_ids.begin(), sorted_node_ids.end());
 
-      double coordinates[3];
+      real_t coordinates[3];
       for (size_t i = 0; i < sorted_node_ids.size(); i++)
       {
          int node_id = sorted_node_ids[i];
@@ -802,7 +810,7 @@ void ExodusIIWriter::ExtractVertexCoordinates(std::vector<double> & coordx,
    {
       for (int ivertex = 0; ivertex < mesh.GetNV(); ivertex++)
       {
-         double * coordinates = mesh.GetVertex(ivertex);
+         real_t *coordinates = mesh.GetVertex(ivertex);
 
          coordx[ivertex] = coordinates[0];
          coordy[ivertex] = coordinates[1];
@@ -1080,4 +1088,4 @@ void ExodusIIWriter::CheckNodalFESpaceIsSecondOrderH1() const
 
 #endif
 
-}
+}

mesh/mesh.cpp

@@ -32,6 +32,7 @@
 #include <cstring>
 #include <ctime>
 #include <functional>
+#include <set>
 #include <numeric>
 #include <unordered_map>
 #include <unordered_set>
@@ -921,6 +922,63 @@ const FaceGeometricFactors* Mesh::GetFaceGeometricFactors(
    return gf;
 }
 
+const Array<int>& Mesh::GetBdrFaceAttributes() const
+{
+   if (bdr_face_attrs_cache.Size() == 0)
+   {
+      std::unordered_map<int, int> f_to_be;
+      for (int i = 0; i < GetNBE(); ++i)
+      {
+         const int f = GetBdrElementFaceIndex(i);
+         f_to_be[f] = i;
+      }
+      const int nf_bdr = GetNFbyType(FaceType::Boundary);
+      // MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
+      bdr_face_attrs_cache.SetSize(nf_bdr);
+      int f_ind = 0;
+      const int nf = GetNumFaces();
+      for (int f = 0; f < nf; ++f)
+      {
+         if (!GetFaceInformation(f).IsOfFaceType(FaceType::Boundary))
+         {
+            continue;
+         }
+         int attribute = -1; // default value
+         auto iter = f_to_be.find(f);
+         if (iter != f_to_be.end())
+         {
+            const int be = iter->second;
+            attribute = GetBdrAttribute(be);
+         }
+         else
+         {
+            // If a boundary face does not correspond to the a boundary element,
+            // we assign it the default attribute of -1.
+         }
+         bdr_face_attrs_cache[f_ind] = attribute;
+         ++f_ind;
+      }
+   }
+   return bdr_face_attrs_cache;
+}
+
+const Array<int>& Mesh::GetElementAttributes() const
+{
+   if (elem_attrs_cache.Size() == 0)
+   {
+      // re-compute cache
+      elem_attrs_cache.SetSize(GetNE());
+      elem_attrs_cache.HostWrite();
+      for (int i = 0; i < GetNE(); ++i)
+      {
+         elem_attrs_cache[i] = GetAttribute(i);
+         MFEM_ASSERT(elem_attrs_cache[i] > 0,
+                     "Negative attribute on element " << i);
+      }
+   }
+   return elem_attrs_cache;
+}
+
 void Mesh::DeleteGeometricFactors()
 {
    for (int i = 0; i < geom_factors.Size(); i++)
@@ -1804,6 +1862,8 @@ void Mesh::Destroy()
    TetMemory.Clear();
 #endif
 
+   elem_attrs_cache.DeleteAll();
+   bdr_face_attrs_cache.DeleteAll();
    attributes.DeleteAll();
    bdr_attributes.DeleteAll();
 }
@@ -1817,36 +1877,48 @@ void Mesh::ResetLazyData()
    DeleteGeometricFactors();
    nbInteriorFaces = -1;
    nbBoundaryFaces = -1;
+   // set size to 0 so re-computations can potentially avoid a new allocation
+   bdr_face_attrs_cache.SetSize(0);
+   elem_attrs_cache.SetSize(0);
 }
 
-void Mesh::SetAttributes()
+void Mesh::SetAttributes(bool elem_attrs_changed, bool bdr_face_attrs_changed)
 {
-   Array<int> attribs;
+   if (bdr_face_attrs_changed)
+   {
+      bdr_face_attrs_cache.SetSize(0); // Invalidate the cache
 
-   attribs.SetSize(GetNBE());
-   for (int i = 0; i < attribs.Size(); i++)
-   {
-      attribs[i] = GetBdrAttribute(i);
-   }
-   attribs.Sort();
-   attribs.Unique();
-   attribs.Copy(bdr_attributes);
-   if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0)
-   {
-      MFEM_WARNING("Non-positive attributes on the boundary!");
+      // Get sorted list of unique boundary element attributes
+      std::set<int> attribs;
+      for (int i = 0; i < GetNBE(); i++)
+      {
+         attribs.emplace(GetBdrAttribute(i));
+      }
+
+      bdr_attributes.SetSize(attribs.size());
+      bdr_attributes.HostWrite();
+      std::copy(attribs.begin(), attribs.end(), bdr_attributes.begin());
+      if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0)
+      {
+         MFEM_WARNING("Non-positive attributes on the boundary!");
+      }
    }
 
-   attribs.SetSize(GetNE());
-   for (int i = 0; i < attribs.Size(); i++)
+   if (elem_attrs_changed)
    {
-      attribs[i] = GetAttribute(i);
-   }
-   attribs.Sort();
-   attribs.Unique();
-   attribs.Copy(attributes);
-   if (attributes.Size() > 0 && attributes[0] <= 0)
-   {
-      MFEM_WARNING("Non-positive attributes in the domain!");
+      // Re-compute the attributes cache
+      elem_attrs_cache.SetSize(0);
+      GetElementAttributes();
+      // Get sorted list of unique element attributes
+      std::set<int> attribs(elem_attrs_cache.begin(), elem_attrs_cache.end());
+      attributes.SetSize(attribs.size());
+      attributes.HostWrite();
+      std::copy(attribs.begin(), attribs.end(), attributes.begin());
+
+      if (attributes.Size() > 0 && attributes[0] <= 0)
+      {
+         MFEM_WARNING("Non-positive attributes in the domain!");
+      }
    }
 }
 
@@ -2994,6 +3066,7 @@ void Mesh::DoNodeReorder(DSTable *old_v_to_v, Table *old_elem_vert)
    const int num_edge_dofs = old_dofs.Size();
 
    // Save the original nodes
+   Nodes->HostReadWrite(); // for "(*Nodes)() = "
    const Vector onodes = *Nodes;
 
    // vertex dofs do not need to be moved
@@ -4453,6 +4526,10 @@ Mesh::Mesh(const Mesh &mesh, bool copy_nodes)
       Nodes = mesh.Nodes;
       own_nodes = 0;
    }
+
+   // copy attribute caches
+   elem_attrs_cache = mesh.elem_attrs_cache;
+   bdr_face_attrs_cache = mesh.bdr_face_attrs_cache;
 }
 
 Mesh::Mesh(Mesh &&mesh) : Mesh()
@@ -7720,6 +7797,12 @@ void Mesh::GetBdrElementAdjacentElement2(
 void Mesh::SetAttribute(int i, int attr)
 {
   elements[i]->SetAttribute(attr);
+  if (elem_attrs_cache.Size() == GetNE())
+  {
+     // update the existing cache instead of deleting it
+     elem_attrs_cache.HostReadWrite();
+     elem_attrs_cache[i] = attr;
+  }
   if (ncmesh) ncmesh->SetAttribute(i, attr);
 }
 
@@ -10849,6 +10932,10 @@ void Mesh::Swap(Mesh& other, bool non_geometry)
       mfem::Swap(nodes_sequence, other.nodes_sequence);
       mfem::Swap(last_operation, other.last_operation);
    }
+
+   // copy attribute caches
+   mfem::Swap(elem_attrs_cache, other.elem_attrs_cache);
+   mfem::Swap(bdr_face_attrs_cache, other.bdr_face_attrs_cache);
 }
 
 void Mesh::GetElementData(const Array<Element*> &elem_array, int geom,
@@ -13256,7 +13343,7 @@ void Mesh::ScaleElements(real_t sf)
    delete [] vn;
 }
 
-void Mesh::Transform(void (*f)(const Vector&, Vector&))
+void Mesh::Transform(std::function<void(const Vector &, Vector&)> f)
 {
    // TODO: support for different new spaceDim.
    if (Nodes == NULL)
@@ -13269,7 +13356,7 @@ void Mesh::Transform(void (*f)(const Vector&, Vector&))
             vold(j) = vertices[i](j);
          }
          vnew.SetData(vertices[i]());
-         (*f)(vold, vnew);
+         f(vold, vnew);
       }
    }
    else

mesh/mesh.hpp

@@ -107,6 +107,11 @@ protected:
    Array<Element *> boundary;
    Array<Element *> faces;
 
+   /// internal cache for element attributes
+   mutable Array<int> elem_attrs_cache;
+   /// internal cache for boundary element attributes
+   mutable Array<int> bdr_face_attrs_cache;
+
    /** @brief This structure stores the low level information necessary to
        interpret the configuration of elements on a specific face. This
        information can be accessed using methods like GetFaceElements(),
@@ -588,9 +593,10 @@ protected:
    void Loader(std::istream &input, int generate_edges = 0,
                std::string parse_tag = "");
 
-   /** If NURBS mesh, write NURBS format. If NCMesh, write mfem v1.1 format.
-       If section_delimiter is empty, write mfem v1.0 format. Otherwise, write
-       mfem v1.2 format with the given section_delimiter at the end.
+   /** @brief If NURBS mesh, write NURBS format. If NCMesh, write mfem v1.1
+       format. If section_delimiter is empty, write mfem v1.0 format. Otherwise,
+       write mfem v1.2 format with the given section_delimiter at the end.
+
        If @a comments is non-empty, it will be printed after the first line of
        the file, and each line should begin with '#'. */
    void Printer(std::ostream &os = mfem::out,
@@ -1121,13 +1127,14 @@ public:
        Mesh vertices or nodes are set. */
    virtual void Finalize(bool refine = false, bool fix_orientation = false);
 
-   /// @brief Determine the sets of unique attribute values in domain and
-   /// boundary elements.
+   /// @brief Determine the sets of unique attribute values in domain if @a
+   /// elem_attrs_changed and boundary elements if @a bdr_face_attrs_changed.
    ///
    /// Separately scan the domain and boundary elements to generate unique,
    /// sorted sets of the element attribute values present in the mesh and
    /// store these in the Mesh::attributes and Mesh::bdr_attributes arrays.
-   virtual void SetAttributes();
+   virtual void SetAttributes(bool elem_attrs_changed = true,
+                              bool bdr_face_attrs_changed = true);
 
    /// Check (and optionally attempt to fix) the orientation of the elements
    /** @param[in] fix_it  If `true`, attempt to fix the orientations of some
@@ -2253,7 +2260,7 @@ public:
    void ScaleSubdomains (real_t sf);
    void ScaleElements (real_t sf);
 
-   void Transform(void (*f)(const Vector&, Vector&));
+   void Transform(std::function<void(const Vector &, Vector&)> f);
    void Transform(VectorCoefficient &deformation);
 
    /** @brief This function should be called after the mesh node coordinates
@@ -2266,6 +2273,35 @@ public:
        method does not modify the nodes. */
    void NodesUpdated() { DeleteGeometricFactors(); }
 
+   /// @brief Returns the attributes for all elements in this mesh. The i'th
+   /// entry of the array is the attribute of the i'th element of the mesh.
+   ///
+   /// The returned array points to an internal object that may be invalidated
+   /// by mesh operations such as refinement or any element attributes are
+   /// modified. Since not all such modifications can be tracked by the Mesh
+   /// class (e.g. if a user calls GetElement() then changes the element
+   /// attribute directly), one needs to account for such changes by calling the
+   /// method SetAttributes().
+   const Array<int>& GetElementAttributes() const;
+
+   /// @brief Returns the attributes for all boundary elements in this mesh.
+   ///
+   /// The face restriction will give "face E-vectors" on the boundary that
+   /// are numbered in the order of the faces of mesh. This numbering will be
+   /// different than the numbering of the boundary elements. We compute
+   /// mappings so that the array `bdr_attributes[i]` gives the boundary
+   /// attribute of the `i`th boundary face in the mesh face order.
+   /// Attributes <= 0 indicate there is no boundary element and should be
+   /// skipped.
+   ///
+   /// The returned array points to an internal object that may be invalidated
+   /// by mesh operations such as refinement or any element attributes are
+   /// modified. Since not all such modifications can be tracked by the Mesh
+   /// class (e.g. if a user calls GetElement() then changes the element
+   /// attribute directly), one needs to account for such changes by calling the
+   /// method SetAttributes().
+   const Array<int>& GetBdrFaceAttributes() const;
+
    /// @}
 
    /// @anchor mfem_Mesh_gf_nodes
@@ -2482,10 +2518,12 @@ public:
    /// Print the mesh to the given stream using Netgen/Truegrid format.
    virtual void PrintXG(std::ostream &os = mfem::out) const;
 
-   /// Print the mesh to the given stream using the default MFEM mesh format.
-   /// \see mfem::ofgzstream() for on-the-fly compression of ascii outputs. If
-   /// @a comments is non-empty, it will be printed after the first line of the
-   /// file, and each line should begin with '#'.
+   /** @brief Print the mesh to the given stream using the default MFEM mesh
+       format.
+
+       \see mfem::ofgzstream() for on-the-fly compression of ascii outputs. If
+       @a comments is non-empty, it will be printed after the first line of the
+       file, and each line should begin with '#'. */
    virtual void Print(std::ostream &os = mfem::out,
                       const std::string &comments = "") const
    { Printer(os, "", comments); }
@@ -2537,7 +2575,7 @@ public:
 
 #ifdef MFEM_USE_NETCDF
    /// @brief Export a mesh to an Exodus II file.
-   void PrintExodusII(const std::string fpath);
+   void PrintExodusII(const std::string &fpath);
 #endif
 
    /** @brief Prints the mesh with boundary elements given by the boundary of

mesh/mesh_readers.cpp

@@ -802,21 +802,11 @@ struct BufferReader : BufferReaderBase
       {
          // Each "data block" is preceded by a header that is either UInt32 or
          // UInt64. The rest of the data follows.
-         uint64_t data_size;
-         if (header_type == UINT32_HEADER)
-         {
-            uint32_t *data_size_32 = (uint32_t *)header_buf;
-            data_size = *data_size_32;
-         }
-         else
-         {
-            uint64_t *data_size_64 = (uint64_t *)header_buf;
-            data_size = *data_size_64;
-         }
-         MFEM_VERIFY(sizeof(F)*n == data_size, "AppendedData: wrong data size");
+         MFEM_VERIFY(sizeof(F)*n == ReadHeaderEntry(header_buf),
+                     "AppendedData: wrong data size");
       }
 
-      if (std::is_same<T, F>::value)
+      if (std::is_same_v<T, F>)
       {
          // Special case: no type conversions necessary, so can just memcpy
          memcpy(dest, buf, sizeof(T)*n);

mesh/nurbs.cpp

@@ -9,8 +9,13 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "mesh_headers.hpp"
-#include "../fem/fem.hpp"
+#include "nurbs.hpp"
+
+#include "point.hpp"
+#include "segment.hpp"
+#include "quadrilateral.hpp"
+#include "hexahedron.hpp"
+#include "../fem/gridfunc.hpp"
 #include "../general/text.hpp"
 
 #include <fstream>
@@ -33,6 +38,7 @@ KnotVector::KnotVector(istream &input)
 
    knot.Load(input, NumOfControlPoints + Order + 1);
    GetElements();
+   coarse = false;
 }
 
 KnotVector::KnotVector(int order, int NCP)
@@ -41,12 +47,13 @@ KnotVector::KnotVector(int order, int NCP)
    NumOfControlPoints = NCP;
    knot.SetSize(NumOfControlPoints + Order + 1);
    NumOfElements = 0;
+   coarse = false;
 
    knot = -1.;
 }
 
 KnotVector::KnotVector(int order, const Vector& intervals,
-                       const Array<int>& continuity )
+                       const Array<int>& continuity)
 {
    // NOTE: This may need to be generalized to support periodicity
    // in the future.
@@ -86,6 +93,7 @@ KnotVector::KnotVector(int order, const Vector& intervals,
          ++NumOfElements;
       }
    }
+   coarse = false;
 }
 
 KnotVector &KnotVector::operator=(const KnotVector &kv)
@@ -143,7 +151,7 @@ void KnotVector::UniformRefinement(Vector &newknots, int rf) const
       {
          for (int m = 1; m < rf; ++m)
          {
-            newknots(j) = m * h * (knot(i) + knot(i+1));
+            newknots(j) = ((1.0 - (m * h)) * knot(i)) + (m * h * knot(i+1));
             j++;
          }
       }
@@ -332,7 +340,7 @@ void KnotVector::PrintFunctions(std::ostream &os, int samples) const
    }
 }
 
-// Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
+// Routine from "The NURBS Book" - 2nd ed - Piegl and Tiller
 // Algorithm A2.2 p. 70
 void KnotVector::CalcShape(Vector &shape, int i, real_t xi) const
 {
@@ -359,7 +367,7 @@ void KnotVector::CalcShape(Vector &shape, int i, real_t xi) const
    }
 }
 
-// Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
+// Routine from "The NURBS Book" - 2nd ed - Piegl and Tiller
 // Algorithm A2.3 p. 72
 void KnotVector::CalcDShape(Vector &grad, int i, real_t xi) const
 {
@@ -417,7 +425,7 @@ void KnotVector::CalcDShape(Vector &grad, int i, real_t xi) const
    }
 }
 
-// Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
+// Routine from "The NURBS Book" - 2nd ed - Piegl and Tiller
 // Algorithm A2.3 p. 72
 void KnotVector::CalcDnShape(Vector &gradn, int n, int i, real_t xi) const
 {
@@ -537,11 +545,11 @@ void KnotVector::FindMaxima(Array<int> &ks, Vector &xi, Vector &u) const
          int i = j - d;
          if (isElement(i))
          {
-            arg1 = 1e-16;
+            arg1 = std::numeric_limits<real_t>::epsilon() / 2_r;
             CalcShape(shape, i, arg1);
             max1 = shape[d];
 
-            arg2 = 1-(1e-16);
+            arg2 = 1_r - arg1;
             CalcShape(shape, i, arg2);
             max2 = shape[d];
 
@@ -579,9 +587,9 @@ void KnotVector::FindMaxima(Array<int> &ks, Vector &xi, Vector &u) const
    }
 }
 
-// Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
+// Routine from "The NURBS Book" - 2nd ed - Piegl and Tiller
 // Algorithm A9.1 p. 369
-void KnotVector::FindInterpolant(Array<Vector*> &x)
+void KnotVector::FindInterpolant(Array<Vector*> &x, bool reuse_inverse)
 {
    int order = GetOrder();
    int ncp = GetNCP();
@@ -589,29 +597,93 @@ void KnotVector::FindInterpolant(Array<Vector*> &x)
    // Find interpolation points
    Vector xi_args, u_args;
    Array<int> i_args;
-   FindMaxima(i_args,xi_args, u_args);
+   FindMaxima(i_args, xi_args, u_args);
 
    // Assemble collocation matrix
-   Vector shape(order+1);
-   DenseMatrix A(ncp,ncp);
-   A = 0.0;
+#ifdef MFEM_USE_LAPACK
+   // If using LAPACK, we use banded matrix storage (order + 1 nonzeros per row).
+   // Find banded structure of matrix.
+   int KL = 0; // Number of subdiagonals
+   int KU = 0; // Number of superdiagonals
    for (int i = 0; i < ncp; i++)
    {
-      CalcShape(shape, i_args[i], xi_args[i]);
       for (int p = 0; p < order+1; p++)
       {
-         A(i,i_args[i] + p) =  shape[p];
+         const int col = i_args[i] + p;
+         if (col < i)
+         {
+            KL = std::max(KL, i - col);
+         }
+         else if (i < col)
+         {
+            KU = std::max(KU, col - i);
+         }
       }
    }
 
-   // Solve problems
-   A.Invert();
+   const int LDAB = (2*KL) + KU + 1;
+   const int N = ncp;
+
+   fact_AB.SetSize(LDAB, N);
+#else
+   // Without LAPACK, we store and invert a DenseMatrix (inefficient).
+   if (!reuse_inverse)
+   {
+      A_coll_inv.SetSize(ncp, ncp);
+      A_coll_inv = 0.0;
+   }
+#endif
+
+   Vector shape(order+1);
+
+   if (!reuse_inverse) // Set collocation matrix entries
+   {
+      for (int i = 0; i < ncp; i++)
+      {
+         CalcShape(shape, i_args[i], xi_args[i]);
+         for (int p = 0; p < order+1; p++)
+         {
+            const int j = i_args[i] + p;
+#ifdef MFEM_USE_LAPACK
+            fact_AB(KL+KU+i-j,j) = shape[p];
+#else
+            A_coll_inv(i,j) = shape[p];
+#endif
+         }
+      }
+   }
+
+   // Solve the system
+#ifdef MFEM_USE_LAPACK
+   const int NRHS = x.Size();
+   DenseMatrix B(N, NRHS);
+   for (int j=0; j<NRHS; ++j)
+   {
+      for (int i=0; i<N; ++i) { B(i, j) = (*x[j])[i]; }
+   }
+
+   if (reuse_inverse)
+   {
+      BandedFactorizedSolve(KL, KU, fact_AB, B, false, fact_ipiv);
+   }
+   else
+   {
+      BandedSolve(KL, KU, fact_AB, B, fact_ipiv);
+   }
+
+   for (int j=0; j<NRHS; ++j)
+   {
+      for (int i=0; i<N; ++i) { (*x[j])[i] = B(i, j); }
+   }
+#else
+   if (!reuse_inverse) { A_coll_inv.Invert(); }
    Vector tmp;
-   for (int i= 0; i < x.Size(); i++)
+   for (int i = 0; i < x.Size(); i++)
    {
       tmp = *x[i];
-      A.Mult(tmp,*x[i]);
+      A_coll_inv.Mult(tmp, *x[i]);
    }
+#endif
 }
 
 int KnotVector::findKnotSpan(real_t u) const
@@ -1413,7 +1485,7 @@ void NURBSPatch::DegreeElevate(int t)
    }
 }
 
-// Routine from "The NURBS book" - 2nd ed - Piegl and Tiller
+// Routine from "The NURBS Book" - 2nd ed - Piegl and Tiller
 void NURBSPatch::DegreeElevate(int dir, int t)
 {
    if (dir >= kv.Size() || dir < 0)
@@ -1431,8 +1503,8 @@ void NURBSPatch::DegreeElevate(int dir, int t)
    KnotVector &oldkv = *kv[dir];
    oldkv.GetElements();
 
-   NURBSPatch *newpatch = new NURBSPatch(this, dir, oldkv.GetOrder() + t,
-                                         oldkv.GetNCP() + oldkv.GetNE()*t);
+   auto *newpatch = new NURBSPatch(this, dir, oldkv.GetOrder() + t,
+                                   oldkv.GetNCP() + oldkv.GetNE()*t);
    NURBSPatch &newp  = *newpatch;
    KnotVector &newkv = *newp.GetKV(dir);
 
@@ -2377,7 +2449,7 @@ NURBSExtension::NURBSExtension(Mesh *mesh_array[], int num_pieces)
 }
 
 NURBSExtension::NURBSExtension(const Mesh *patch_topology,
-                               const Array<const NURBSPatch*> patches_)
+                               const Array<const NURBSPatch*> &patches_)
 {
    // Basic topology checks
    MFEM_VERIFY(patches_.Size() > 0, "Must have at least one patch");
@@ -4587,7 +4659,7 @@ void NURBSExtension::KnotInsert(Array<Vector *> &kv)
 
             // Flip vector
             int size = pkvc[d]->Size();
-            int ns = ceil(size/2.0);
+            int ns = static_cast<int>(ceil(size/2.0));
             for (int j = 0; j < ns; j++)
             {
                real_t tmp = apb - pkvc[d]->Elem(j);
@@ -4647,7 +4719,7 @@ void NURBSExtension::KnotRemove(Array<Vector *> &kv, real_t tol)
 
             // Flip vector
             int size = pkvc[d]->Size();
-            int ns = ceil(size/2.0);
+            int ns = static_cast<int>(ceil(size/2.0));
             for (int j = 0; j < ns; j++)
             {
                real_t tmp = apb - pkvc[d]->Elem(j);

mesh/nurbs.hpp

@@ -22,7 +22,6 @@
 #include "../general/communication.hpp"
 #endif
 #include <iostream>
-#include <set>
 
 namespace mfem
 {
@@ -55,7 +54,7 @@ protected:
 
 public:
    /// Create an empty KnotVector.
-   KnotVector() { }
+   KnotVector() = default;
 
    /** @brief Create a KnotVector by reading data from stream @a input. Two
        integers are read, for order and number of control points. */
@@ -74,7 +73,7 @@ public:
        polynomial degree). Periodicity is not supported.
    */
    KnotVector(int order, const Vector& intervals,
-              const Array<int>& continuity );
+              const Array<int>& continuity);
 
    /// Copy constructor.
    KnotVector(const KnotVector &kv) { (*this) = kv; }
@@ -144,8 +143,13 @@ public:
    /** @brief Global curve interpolation through the points @a x (overwritten).
        @a x is an array with the length of the spatial dimension containing
        vectors with spatial coordinates. The control points of the interpolated
-       curve are returned in @a x in the same form. */
-   void FindInterpolant(Array<Vector*> &x);
+       curve are returned in @a x in the same form.
+
+       The inverse of the collocation matrix, used in the interpolation, is
+       stored for repeated calls and used if @a reuse_inverse is true. Reuse is
+       valid only if this KnotVector has not changed since the initial call with
+       @a reuse_inverse false. */
+   void FindInterpolant(Array<Vector*> &x, bool reuse_inverse = false);
 
    /** Set @a diff, comprised of knots in @a kv not contained in this KnotVector.
        @a kv must be of the same order as this KnotVector. The current
@@ -203,6 +207,14 @@ public:
    /** Flag to indicate whether the KnotVector has been coarsened, which means
        it is ready for non-nested refinement. */
    bool coarse;
+
+#ifdef MFEM_USE_LAPACK
+   // Data for reusing banded matrix factorization in FindInterpolant().
+   DenseMatrix fact_AB; /// Banded matrix factorization
+   Array<int> fact_ipiv; /// Row pivot indices
+#else
+   DenseMatrix A_coll_inv; /// Collocation matrix inverse
+#endif
 };
 
 
@@ -286,7 +298,7 @@ public:
        includes the weight. The array of control point coordinates stores each
        point's coordinates contiguously, and points are ordered in a standard
        ijk grid ordering. */
-   NURBSPatch(Array<const KnotVector *> &kv_,  int dim_,
+   NURBSPatch(Array<const KnotVector *> &kv_, int dim_,
               const real_t* control_points);
 
    /// Constructor for a patch of dimension equal to the size of @a kv.
@@ -701,7 +713,8 @@ public:
 
    NURBSExtension(Mesh *mesh_array[], int num_pieces);
 
-   NURBSExtension(const Mesh *patch_topology, const Array<const NURBSPatch*> p);
+   NURBSExtension(const Mesh *patch_topology,
+                  const Array<const NURBSPatch*> &patches_);
 
    /// Copy assignment not supported.
    NURBSExtension& operator=(const NURBSExtension&) = delete;

mesh/pmesh.cpp

@@ -1590,21 +1590,27 @@ void ParMesh::DistributeAttributes(Array<int> &attr)
    delete [] glb_attr_marker;
 }
 
-void ParMesh::SetAttributes()
+void ParMesh::SetAttributes(bool elem_attrs_changed, bool bdr_attrs_changed)
 {
    // Determine the attributes occurring in local interior and boundary elements
-   Mesh::SetAttributes();
+   Mesh::SetAttributes(elem_attrs_changed, bdr_attrs_changed);
 
-   DistributeAttributes(bdr_attributes);
-   if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0)
+   if (bdr_attrs_changed)
    {
-      MFEM_WARNING("Non-positive boundary element attributes found!");
+      DistributeAttributes(bdr_attributes);
+      if (bdr_attributes.Size() > 0 && bdr_attributes[0] <= 0)
+      {
+         MFEM_WARNING("Non-positive boundary element attributes found!");
+      }
    }
 
-   DistributeAttributes(attributes);
-   if (attributes.Size() > 0 && attributes[0] <= 0)
+   if (elem_attrs_changed)
    {
-      MFEM_WARNING("Non-positive element attributes found!");
+      DistributeAttributes(attributes);
+      if (attributes.Size() > 0 && attributes[0] <= 0)
+      {
+         MFEM_WARNING("Non-positive element attributes found!");
+      }
    }
 }
 
@@ -3132,11 +3138,12 @@ void ParMesh::GetFaceNbrElementTransformation(
          pNodes->ParFESpace()->GetFaceNbrElementVDofs(FaceNo, vdofs);
          int n = vdofs.Size()/spaceDim;
          pointmat.SetSize(spaceDim, n);
+         pNodes->FaceNbrData().HostRead();
          for (int k = 0; k < spaceDim; k++)
          {
             for (int j = 0; j < n; j++)
             {
-               pointmat(k,j) = (pNodes->FaceNbrData())(vdofs[n*k+j]);
+               pointmat(k,j) = AsConst(pNodes->FaceNbrData())(vdofs[n*k+j]);
             }
          }
 

mesh/pmesh.hpp

@@ -394,7 +394,8 @@ public:
 
    void Finalize(bool refine = false, bool fix_orientation = false) override;
 
-   void SetAttributes() override;
+   void SetAttributes(bool elem_attrs_changed = true,
+                      bool bdr_attrs_changed = true) override;
 
    /// Checks if any rank in the mesh has boundary elements
    bool HasBoundaryElements() const override;

mesh/submesh/submesh_utils.cpp

@@ -257,15 +257,7 @@ template <typename SubMeshT>
 void AddBoundaryElements(SubMeshT &mesh,
                          const std::unordered_map<int,int> &lface_to_boundary_attribute)
 {
-   mesh.Dimension();
-   const int num_codim_1 = [&mesh]()
-   {
-      auto Dim = mesh.Dimension();
-      if (Dim == 1) { return mesh.GetNV(); }
-      else if (Dim == 2) { return mesh.GetNEdges(); }
-      else if (Dim == 3) { return mesh.GetNFaces(); }
-      else { MFEM_ABORT("Invalid dimension."); return -1; }
-   }();
+   const int num_codim_1 = mesh.GetNumFaces();
 
    if (mesh.Dimension() == 3)
    {

mesh/vtkhdf.cpp

@@ -84,7 +84,7 @@ void VTKHDF::EnsureSteps()
 }
 
 hid_t VTKHDF::EnsureDataset(hid_t f, const std::string &name, hid_t type,
-                            int ndims)
+                            Dims &dims)
 {
    const char *name_c = name.c_str();
 
@@ -94,20 +94,23 @@ hid_t VTKHDF::EnsureDataset(hid_t f, const std::string &name, hid_t type,
    if (status == 0)
    {
       // Dataset does not exist, create it.
-      Dims dims(ndims);
-      Dims maxdims(ndims, H5S_UNLIMITED);
-      const hid_t fspace = H5Screate_simple(ndims, dims, maxdims);
+      const int ndims = dims.ndims;
+      // The dataset is allowed to grow in the first dimension, but is fixed
+      // in size in all other dimesions; the maximum dataset size is same as
+      // dims, but unlimited in first dimension.
+      Dims max_dims = dims;
+      max_dims[0] = H5S_UNLIMITED;
+      const hid_t fspace = H5Screate_simple(ndims, dims, max_dims);
 
       Dims chunk(ndims);
       size_t chunk_size_bytes = 1024 * 1024 / 2; // 0.5 MB
       const size_t t_bytes = H5Tget_size(type);
       for (int i = 1; i < ndims; ++i)
       {
-         chunk[i] = 16;
-         chunk_size_bytes /= 16;
+         chunk[i] = dims[i];
+         chunk_size_bytes /= dims[i];
       }
       chunk[0] = chunk_size_bytes / t_bytes;
-      for (int i = 1; i < ndims; ++i) { chunk[i] = 16; }
       const hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
       H5Pset_chunk(dcpl, ndims, chunk);
       if (compression_level >= 0)
@@ -124,7 +127,19 @@ hid_t VTKHDF::EnsureDataset(hid_t f, const std::string &name, hid_t type,
    else if (status > 0)
    {
       // Dataset exists, open it.
-      return H5Dopen2(f, name_c, H5P_DEFAULT);
+      const hid_t d = H5Dopen2(f, name_c, H5P_DEFAULT);
+
+      // Resize the dataset, set dims to its new size.
+      Dims old_dims(dims.ndims);
+      const hid_t dspace = H5Dget_space(d);
+      const int ndims_dset = H5Sget_simple_extent_ndims(dspace);
+      MFEM_VERIFY(ndims_dset == dims.ndims, "");
+      H5Sget_simple_extent_dims(dspace, old_dims, NULL);
+      H5Sclose(dspace);
+      dims[0] += old_dims[0];
+      H5Dset_extent(d, dims);
+
+      return d;
    }
    else
    {
@@ -160,27 +175,13 @@ void VTKHDF::AppendParData(hid_t f, const std::string &name, hsize_t locsize,
                            hsize_t offset, Dims globsize, T *data)
 {
    const int ndims = globsize.ndims;
-   const hid_t d = EnsureDataset(f, name, GetTypeID<T>(), ndims);
-
-   // Resize the dataset, set dims to its new size.
-   hsize_t old_size;
-   Dims dims(ndims);
-   {
-      const hid_t dspace = H5Dget_space(d);
-      const int ndims_dset = H5Sget_simple_extent_ndims(dspace);
-      MFEM_VERIFY(ndims_dset == ndims, "");
-      H5Sget_simple_extent_dims(dspace, dims, NULL);
-      H5Sclose(dspace);
-      old_size = dims[0];
-      dims[0] += globsize[0];
-      for (int i = 1; i < ndims; ++i) { dims[i] = globsize[i]; }
-      H5Dset_extent(d, dims);
-   }
+   Dims dims = globsize;
+   const hid_t d = EnsureDataset(f, name, GetTypeID<T>(), dims);
 
    // Write the new entry.
    const hid_t dspace = H5Dget_space(d);
    Dims start(ndims);
-   start[0] = old_size + offset;
+   start[0] = dims[0] - globsize[0] + offset;
    Dims count(ndims);
    count[0] = locsize;
    for (int i = 1; i < ndims; ++i) { count[i] = globsize[i]; }
@@ -334,14 +335,14 @@ void VTKHDF::Truncate(const real_t t)
    }
 
    // Index of found time index (may be 'one-past-the-end' if not found)
-   const int i = std::distance(tvals.begin(), it);
+   const ptrdiff_t i = std::distance(tvals.begin(), it);
 
    // Only truncate if needed
    const bool truncate = it != tvals.end();
 
    // Number of steps we are keeping
    nsteps = i;
-   H5LTset_attribute_int(vtk, "Steps", "NSteps", &nsteps, 1);
+   H5LTset_attribute_ulong(vtk, "Steps", "NSteps", &nsteps, 1);
 
    // We want to continue writing immediately after step 'i - 1'. If i = 0,
    // then this is at the beginning of the file, and the offsets do not need
@@ -509,7 +510,7 @@ void VTKHDF::UpdateSteps(real_t t)
 
    // Set the NSteps attribute
    ++nsteps;
-   H5LTset_attribute_int(steps, ".", "NSteps", &nsteps, 1);
+   H5LTset_attribute_ulong(steps, ".", "NSteps", &nsteps, 1);
 
    AppendValue(steps, "Values", t);
    AppendValue(steps, "PartOffsets", part_offset);
@@ -618,16 +619,16 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
 
          for (int i = 0; i < pmat.Width(); i++)
          {
-            points.push_back(pmat(0,i));
-            if (pmat.Height() > 1) { points.push_back(pmat(1,i)); }
+            points.push_back(FP_T(pmat(0,i)));
+            if (pmat.Height() > 1) { points.push_back(FP_T(pmat(1,i))); }
             else { points.push_back(0.0); }
-            if (pmat.Height() > 2) { points.push_back(pmat(2,i)); }
+            if (pmat.Height() > 2) { points.push_back(FP_T(pmat(2,i))); }
             else { points.push_back(0.0); }
          }
       }
    }
 
-   const hsize_t ne_0 = mesh.GetNE();
+   const int ne_0 = mesh.GetNE();
    const hsize_t ne = high_order ? ne_0 : ne_ref;
 
    AppendParData(vtk, "NumberOfPoints", 1, mpi_rank, mpi_dims, &np);
@@ -657,7 +658,7 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          if (high_order)
          {
             Array<int> local_connectivity;
-            for (size_t e = 0; e < ne; ++e)
+            for (int e = 0; e < int(ne); ++e)
             {
                offsets[e] = off;
                const Geometry::Type geom = mesh.GetElementGeometry(e);
@@ -675,7 +676,7 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          {
             int off_0 = 0;
             int e_ref = 0;
-            for (hsize_t e = 0; e < ne_0; ++e)
+            for (int e = 0; e < ne_0; ++e)
             {
                const Geometry::Type geom = mesh.GetElementGeometry(e);
                const int nv = get_nv(e);
@@ -714,12 +715,13 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          const int *vtk_geom_map =
             high_order ? VTKGeometry::HighOrderMap : VTKGeometry::Map;
          int e_ref = 0;
-         for (hsize_t e = 0; e < ne_0; ++e)
+         for (int e = 0; e < ne_0; ++e)
          {
-            const int ne_ref = get_ne_ref(e, ref_0);
-            for (int i = 0; i < ne_ref; ++i, ++e_ref)
+            const int ne_ref_e = get_ne_ref(e, ref_0);
+            for (int i = 0; i < ne_ref_e; ++i, ++e_ref)
             {
-               cell_types[e_ref] = vtk_geom_map[mesh.GetElementGeometry(e)];
+               cell_types[e_ref] = static_cast<unsigned char>(
+                                      vtk_geom_map[mesh.GetElementGeometry(e)]);
             }
          }
          AppendParData(vtk, "Types", ne, e_offset, Dims({ne_total}),
@@ -732,11 +734,11 @@ void VTKHDF::SaveMesh(const Mesh &mesh, bool high_order, int ref)
          EnsureGroup("CellData", cell_data);
          std::vector<int> attributes(ne);
          hsize_t e_ref = 0;
-         for (hsize_t e = 0; e < ne_0; ++e)
+         for (int e = 0; e < ne_0; ++e)
          {
             const int attr = mesh.GetAttribute(e);
-            const int ne_ref = get_ne_ref(e, ref_0);
-            for (int i = 0; i < ne_ref; ++i, ++e_ref)
+            const int ne_ref_e = get_ne_ref(e, ref_0);
+            for (int i = 0; i < ne_ref_e; ++i, ++e_ref)
             {
                attributes[e_ref] = attr;
             }
@@ -772,7 +774,7 @@ void VTKHDF::SaveGridFunction(const GridFunction &gf, const std::string &name)
       {
          for (int vd = 0; vd < vdim; ++vd)
          {
-            point_values[off] = vec_val(vd, i);
+            point_values[off] = FP_T(vec_val(vd, i));
             ++off;
          }
       }

mesh/vtkhdf.hpp

@@ -76,14 +76,14 @@ private:
    /// Wrapper for storing dataset dimensions (max ndims is 2D in VTKHDF).
    struct Dims
    {
-      static constexpr int MAX_NDIMS = 2;
+      static constexpr size_t MAX_NDIMS = 2;
       std::array<hsize_t, MAX_NDIMS> data = { }; // Zero initialized
       int ndims = 0;
       Dims() = default;
       Dims(int ndims_) : ndims(ndims_) { MFEM_ASSERT(ndims <= MAX_NDIMS, ""); }
       Dims(int ndims_, hsize_t val) : Dims(ndims_) { data.fill(val); }
       template <typename T>
-      Dims(std::initializer_list<T> data_) : Dims(data_.size())
+      Dims(std::initializer_list<T> data_) : Dims(int(data_.size()))
       { std::copy(data_.begin(), data_.end(), data.begin()); }
       operator hsize_t*() { return data.data(); }
       hsize_t &operator[](int i) { return data[i]; }
@@ -97,7 +97,7 @@ private:
    hid_t steps = H5I_INVALID_HID;
 
    /// Number of time steps saved.
-   int nsteps = 0;
+   unsigned long nsteps = 0;
 
    /// Keep track of the offsets into the data arrays at each time step.
    struct Offsets
@@ -123,8 +123,8 @@ private:
    class MeshId
    {
       const Mesh *mesh_ptr = nullptr;
-      int sequence = -1;
-      int nodes_sequence = -1;
+      long sequence = -1;
+      long nodes_sequence = -1;
       bool high_order = true;
       int ref = -1;
    public:
@@ -187,8 +187,10 @@ private:
    /// The rank (number of dimensions) of the dataset is given by @a ndims and
    /// its data type is given by @a type.
    ///
-   /// The dataset will initially have zero size and unlimited maximum size.
-   hid_t EnsureDataset(hid_t f, const std::string &name, hid_t type, int ndims);
+   /// If the dataset does not exist, it will initially have size @a dims.
+   /// Otherwise, it will be resized to append data of size @a dims, and @a dims
+   /// will be set to the new total size.
+   hid_t EnsureDataset(hid_t f, const std::string &name, hid_t type, Dims &dims);
 
    /// @brief Ensure the named group is open, creating it if needed. Set @a
    /// group to the ID.

miniapps/meshing/fit-node-position.cpp

@@ -224,5 +224,6 @@ int main (int argc, char *argv[])
    }
 
    delete metric;
+   delete fec_mesh;
    return 0;
 }

miniapps/nurbs/CMakeLists.txt

@@ -80,6 +80,10 @@ add_mfem_miniapp(nurbs_solenoidal
   LIBRARIES mfem)
   add_dependencies(nurbs_solenoidal copy_miniapps_nurbs_data)
 
+add_mfem_miniapp(nurbs_surface
+  MAIN nurbs_surface.cpp
+  LIBRARIES mfem)
+
 if (MFEM_ENABLE_TESTING)
   add_test(NAME nurbs_ex1_1d_r1_o2_ser
     COMMAND $<TARGET_FILE:nurbs_ex1> -no-vis
@@ -247,6 +251,14 @@ if (MFEM_ENABLE_TESTING)
     COMMAND $<TARGET_FILE:nurbs_solenoidal> -no-vis
         -m ${PROJECT_SOURCE_DIR}/data/cube-nurbs.mesh -r 1 -o 2)
 
+  add_test(NAME nurbs_surface_10_10_10_10_ex1_o3_ser
+    COMMAND $<TARGET_FILE:nurbs_surface> -no-vis
+        -o 3 -nx 10 -ny 10 -fnx 10 -fny 10 -ex 1 -orig)
+
+  add_test(NAME nurbs_surface_10_10_40_40_ex1_o3_ser
+    COMMAND $<TARGET_FILE:nurbs_surface> -no-vis
+        -o 3 -nx 10 -ny 10 -fnx 40 -fny 14 -ex 1)
+
 endif()
 
 if (MFEM_USE_MPI)

miniapps/nurbs/makefile

@@ -21,7 +21,7 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_MINIAPPS = nurbs_ex1 nurbs_patch_ex1 nurbs_ex3 nurbs_ex5 nurbs_ex24 \
-   nurbs_curveint nurbs_printfunc nurbs_solenoidal nurbs_naca_cmesh
+   nurbs_curveint nurbs_printfunc nurbs_solenoidal nurbs_naca_cmesh nurbs_surface
 PAR_MINIAPPS = nurbs_ex1p nurbs_ex11p
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
@@ -158,6 +158,13 @@ nurbs_naca_cmesh-test-seq: nurbs_naca_cmesh
 nurbs_printfunc-test-seq: nurbs_printfunc
 	@$(call mfem-test,$<,, NURBS miniapp)
 
+SURF_ARGS_1 := -o 3 -nx 10 -ny 10 -fnx 10 -fny 10 -ex 1 -orig
+SURF_ARGS_2 := -o 3 -nx 10 -ny 10 -fnx 40 -fny 40 -ex 1
+
+nurbs_surface-test-seq: nurbs_surface
+	@$(call mfem-test,$<,, NURBS miniapp,$(SURF_ARGS_1))
+	@$(call mfem-test,$<,, NURBS miniapp,$(SURF_ARGS_2))
+
 EX1P_ARGS_1 :=
 EX1P_ARGS_2 := -m ../../data/pipe-nurbs-2d.mesh -o 2 -no-ibp
 EX1P_ARGS_3 := -m ../../data/ball-nurbs.mesh -o 2 --weak-bc -r 0
@@ -192,6 +199,6 @@ clean-build:
 
 clean-exec:
 	@rm -f refined.mesh sin-fit.mesh ex5.mesh exsol.mesh mesh.* sol.* mode_*
-	@rm -f naca-cmesh.mesh sol_?.gf
+	@rm -f naca-cmesh.mesh sol_?.gf *-Surface.mesh
 	@rm -rf Example1* Example3* Example5* Solenoidal_* ParaView
 	@rm -rf CurveInt Naca_cmesh glvis_naca-cmesh.mesh solution.dat