Try to fix CI failures on Windows by including windows.h before psapi.h

* class Memory; some methods were still using int for sizes
* class Vector, for size and indexing
* all 1D "forall" macros and function templates
  - HIP cannot launch kernels with >= 2^32 total threads
  - CUDA seems to support kernel launches with >= 2^32 total threads
* class DeviceTensor; individual dimensions still use int, however, the
  total 1D index computation uses bigint
* element and face geometric factors use bigint sizes for memory allocations

Left some FIXME comments to be addressed later.

.github/actions/sanitize/mfem/action.yml

@@ -25,7 +25,7 @@ runs:
   steps:
     - uses: ./.github/actions/sanitize/config
 
-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
       if: ${{env.DEBUG == 'true'}}
       id: debug
       with:

.github/actions/sanitize/restore/action.yml

@@ -36,7 +36,7 @@ runs:
   steps:
     - uses: ./.github/actions/sanitize/config
 
-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
       if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
       id: debug
       with:

.github/actions/sanitize/setup/action.yml

@@ -23,7 +23,7 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - uses: actions/cache/restore@v5  # Cache for LLVM libcxx
+    - uses: actions/cache/restore@v4  # Cache for LLVM libcxx
       with:
         path: ${{env.LLVM_DIR}}
         fail-on-cache-miss: true
@@ -32,14 +32,14 @@ runs:
     - uses: ./.github/actions/sanitize/mpi
       if: ${{inputs.par == 'true'}}
 
-    - uses: actions/cache/restore@v5  # Cache for Hypre
+    - uses: actions/cache/restore@v4  # Cache for Hypre
       if: ${{inputs.par == 'true'}}
       with:
         path: ${{env.HYPRE_DIR}}
         fail-on-cache-miss: true
         key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
 
-    - uses: actions/cache/restore@v5  # Cache for Metis
+    - uses: actions/cache/restore@v4  # Cache for Metis
       if: ${{inputs.par == 'true'}}
       with:
         path: ${{env.METIS_DIR}}
@@ -51,13 +51,13 @@ runs:
       run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
       shell: bash
 
-    - uses: actions/cache/restore@v5  # Cache for LSAN suppression file
+    - uses: actions/cache/restore@v4  # Cache for LSAN suppression file
       with:
         path: ${{env.LSAN_DIR}}
         fail-on-cache-miss: true
         key: build-lsan-suppression-file
 
-    - uses: actions/checkout@v6  # Checkout the repository
+    - uses: actions/checkout@v4  # Checkout the repository
       with:
         path: mfem
         # ref: ${{env.BRANCH}}

.github/workflows/build-container.yml

@@ -43,7 +43,7 @@ jobs:
           remove-docker-images: 'true'
 
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       # It's easier to reference named variables than indexes of the matrix
       - name: Set Environment

.github/workflows/builds-and-tests.yml

@@ -153,7 +153,7 @@ jobs:
       # /home/runner/work/mfem/mfem/mfem
       # Note: Done now to access "install-hypre" and "install-metis" actions.
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           path: ${{ env.MFEM_TOP_DIR }}
           # Fetch the complete history for codecov to access commits ID
@@ -225,7 +225,7 @@ jobs:
       - name: cache hypre
         id: hypre-cache
         if: matrix.mpi == 'par'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.HYPRE_TOP_DIR }}
           key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
@@ -255,7 +255,7 @@ jobs:
       - name: cache metis
         id: metis-cache
         if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.METIS_TOP_DIR }}
           key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
@@ -270,7 +270,7 @@ jobs:
       - name: cache vcpkg (Windows)
         id: vcpkg-cache
         if: matrix.os == 'windows-latest'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: vcpkg_cache
           key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
@@ -295,8 +295,7 @@ jobs:
           export HOMEBREW_NO_INSTALL_CLEANUP=1
           brew update
           brew install enzyme
-          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required.*:.*\(llvm[^ ]*\).*/\1/p')
-          echo "ENZYME_LLVM=$ENZYME_LLVM"
+          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
           LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
           echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV

.github/workflows/codeql-analysis.yml

@@ -40,11 +40,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -57,7 +57,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4
+        uses: github/codeql-action/autobuild@v2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -70,4 +70,4 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4
+        uses: github/codeql-action/analyze@v2

.github/workflows/mfem-analysis.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: checkout MFEM
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           path: mfem
 
@@ -50,7 +50,7 @@ jobs:
 
       - name: Cache Hypre Install
         id: hypre-cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.HYPRE_TOP_DIR }}
           key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5
@@ -65,7 +65,7 @@ jobs:
 
       - name: Cache Metis Install
         id: metis-cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.METIS_TOP_DIR }}
           key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

.github/workflows/repo-check.yml

@@ -38,7 +38,7 @@ jobs:
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: copyright check
         id: copyright
@@ -93,7 +93,7 @@ jobs:
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: get astyle
         run: |
@@ -110,7 +110,7 @@ jobs:
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: get doxygen and graphviz
         run: |
@@ -135,7 +135,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/sanitize-build-hypre.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     name: 2.19.0
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.HYPRE_DIR}}
           key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5

.github/workflows/sanitize-build-libcxx.yml

@@ -27,13 +27,13 @@ jobs:
             llvm_use_sanitizer: "Undefined"
     name: ${{matrix.sanitizer}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
         with:
           NO_FLAGS: true
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.LLVM_DIR}}
           key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}

.github/workflows/sanitize-build-lsan.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     name: lsan.supp
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.LSAN_DIR}}
           key: build-lsan-suppression-file

.github/workflows/sanitize-build-metis.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     name: 4.0.3
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.METIS_DIR}}
           key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5

.github/workflows/sanitize-tests.yml

@@ -28,7 +28,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/mfem
         with:
           par: ${{inputs.par}}
@@ -40,7 +40,7 @@ jobs:
     env:
       ex: ${{inputs.par && 'ex1p' || 'ex1'}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -58,7 +58,7 @@ jobs:
     env:
       exclude: ${{inputs.par && '-E "_ser"' || ''}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -82,7 +82,7 @@ jobs:
     env:
       exclude: ${{inputs.par && '-E "_ser"' || ''}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -107,7 +107,7 @@ jobs:
       run: ${{inputs.par && '-R "_cpu_np"' || ''}}
       exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -131,7 +131,7 @@ jobs:
     env:
       unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -165,7 +165,7 @@ jobs:
       unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
       np: ${{inputs.par && '_np=2' || ''}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:

.gitignore

@@ -443,10 +443,6 @@ miniapps/diag-smoothers/mg-abs-l1-jacobi
 miniapps/contact/contact
 miniapps/contact/ParaView
 
-miniapps/plasma/pic/electrostatic-*
-!miniapps/plasma/pic/electrostatic-*.cpp
-miniapps/plasma/pic/*.csv
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

.gitlab/jobs/dane.yml

@@ -85,8 +85,3 @@ opt_par_gcc_10_pumi:
   extends: .mfem_job_on_dane
   variables:
     SPEC: "%gcc@10.3.1 +pumi"
-
-opt_par_gcc_10_gslib:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +gslib"

.gitlab/jobs/matrix.yml

@@ -63,8 +63,3 @@ opt_mpi_cuda_hypre_cuda_gcc:
   extends: .mfem_job_on_matrix
   variables:
     SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"
-
-opt_mpi_cuda_gcc_gslib:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda +gslib cuda_arch=90 ^hypre+cuda"

.gitlab/scripts/baseline

@@ -32,9 +32,9 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
 if [[ "${MACHINE_NAME}" == "dane" ]]; then
-  srun --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
   echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
   exit 1

CHANGELOG

@@ -11,20 +11,8 @@
 Version 4.9.1 (development)
 ===========================
 
-- Policy for AI-assisted contribution added to CONTRIBUTING.md
-
 Discretization improvements
 ---------------------------
-- Replaced legacy simplex quadrature rules with symmetric positive-weight
-  rules for triangles (orders 0-25) and tetrahedra (orders 0-20). These
-  rules guarantee all-positive weights and interior quadrature points,
-  improving numerical stability. Higher orders fall back to Grundmann-Moller.
-  Triangle rules: Witherden & Vincent, Comput. Math. Appl. 69(10):1232-1241,
-  2015.
-  Tet rules (d=1-13): Witherden & Vincent (ibid).
-  Tet rules (d=14-20): Chuluunbaatar et al., Comput. Math. Appl. 124:89-97,
-  2022.
-
 - Improved the gridfunction projection routines. Projections work for Scalar,
   Vector and VectorFE, also NURBS versions. Optionally different types of
   projections can be selected, default behaviour has not changed.

CMakeLists.txt

@@ -652,8 +652,6 @@ foreach(TPL IN LISTS MFEM_TPLS)
   endif()
 endforeach(TPL)
 
-# reverse to remove the first instance of entries in TPL_LIBRARIES
-# so later duplicates are kept (for dependency ordering)
 list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REVERSE TPL_LIBRARIES)
@@ -1017,7 +1015,5 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
 # Create 'config.mk' from 'config.mk.in' for the build and install locations and
 # define install rules for 'config.mk' and 'test.mk'
 #-------------------------------------------------------------------------------
-if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-  option(MFEM_EXPORT_GPU_CONFIG "Export config.mk for GPU-enabled downstream packages" ON)
-endif()
+
 mfem_export_mk_files()

CONTRIBUTING.md

@@ -24,14 +24,6 @@ must be made under this license.
 Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
 in the MFEM community, you agree to abide by its rules.
 
-## AI Policy
-- Use of AI code generation in MFEM is allowed but must be disclosed, e.g. by
-  selecting the `AI-assisted` label on the PR.
-- By submitting a PR, the author acknowledges that they have reviewed and
-  understand the changes they are proposing.
-- PR authors are still responsible for correctness, licensing, and attribution
-  of all changes.
-
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull

config/cmake/MFEMConfig.cmake.in

@@ -109,10 +109,6 @@ if (MFEM_USE_RAJA)
    find_dependency(RAJA)
 endif()
 
-if (MFEM_USE_UMPIRE)
-   find_dependency(umpire)
-endif()
-
 if (NOT TARGET mfem)
   include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake)
 endif (NOT TARGET mfem)

config/cmake/modules/FindUMPIRE.cmake

@@ -14,12 +14,12 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS
 
-if (NOT umpire_ROOT AND UMPIRE_DIR)
-  set(umpire_ROOT ${UMPIRE_DIR})
+if (NOT umpire_DIR AND UMPIRE_DIR)
+  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
 endif()
 message(STATUS "Looking for UMPIRE ...")
 message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
-message(STATUS "      umpire_ROOT = ${umpire_ROOT}")
+message(STATUS "      umpire_DIR = ${umpire_DIR}")
 find_package(umpire CONFIG)
 set(UMPIRE_FOUND ${umpire_FOUND})
 set(UMPIRE_LIBRARIES "umpire")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -701,6 +701,7 @@ endfunction(mfem_find_library)
 # Extract compile and link options needed by the given target.
 #
 function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
+
   if (NOT TARGET ${Target})
     return()
   endif()
@@ -798,12 +799,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
           # message(STATUS "Lib = ${Lib}")
           # Filter-out generator expressions
           if (NOT ("${Lib}" MATCHES "^\\$"))
-            if(NOT ("${Lib}" STREQUAL "dl"))
-              list(APPEND LinkOpts "${Lib}")
-            else()
-              # for some reason libdl doesn't include the "-l"
-              list(APPEND LinkOpts "-ldl")
-            endif()
+            list(APPEND LinkOpts "${Lib}")
           endif()
         else()
           mfem_get_target_options(${Lib} COpts LOpts)
@@ -892,18 +888,9 @@ function(mfem_export_mk_files)
       set(${var} NO)
     endif()
   endforeach()
-  if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
-    set(MFEM_CXX ${CMAKE_CUDA_COMPILER})
-    if(MFEM_CUDA_COMPILER_IS_NVCC)
-      set(MFEM_HOST_CXX ${CMAKE_CUDA_HOST_COMPILER})
-    else()
-      set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
-    endif()
-  else()
-    # mfem doesn't use enable_language(HIP)
-    set(MFEM_CXX ${CMAKE_CXX_COMPILER})
-    set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
-  endif()
+  # TODO: Add support for MFEM_USE_CUDA=YES
+  set(MFEM_CXX ${CMAKE_CXX_COMPILER})
+  set(MFEM_HOST_CXX ${MFEM_CXX})
   set(MFEM_CPPFLAGS "")
   get_target_property(cxx_std mfem CXX_STANDARD)
   # For now, we ignore the setting of the CXX_EXTENSIONS property. If this
@@ -913,50 +900,6 @@ function(mfem_export_mk_files)
   string(STRIP
          "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
          MFEM_CXXFLAGS)
-  if(MFEM_EXPORT_GPU_CONFIG)
-    if (MFEM_USE_CUDA)
-      set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} ${CMAKE_CUDA_FLAGS}")
-      if (MFEM_CUDA_COMPILER_IS_NVCC)
-        set(MFEM_CXXFLAGS "-x=cu ${MFEM_CXXFLAGS} -ccbin ${CMAKE_CXX_COMPILER} --forward-unknown-to-host-compiler")
-        # The following intentionally hides CUDA deprecation warnings
-        foreach(ENTRY IN LISTS CUDAToolkit_INCLUDE_DIRS)
-          set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -isystem ${ENTRY}")
-        endforeach()
-        if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
-          # architecture flags not part of CMAKE_CUDA_FLAGS
-          if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -arch=${CMAKE_CUDA_ARCHITECTURES}")
-          else()
-            foreach (ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-              set(MFEM_CXXFLAGS
-                "${MFEM_CXXFLAGS} -gencode arch=compute_${ENTRY},code=sm_${ENTRY}")
-            endforeach()
-          endif()
-        endif()
-      else()
-        set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xcuda --cuda-path=${CUDAToolkit_LIBRARY_ROOT}")
-        if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
-          # architecture flags not part of CMAKE_CUDA_FLAGS
-          if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            # TODO: not supported
-          else()
-            foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-              set(MFEM_CXXFLAGS "-cuda-gpu-arch=sm_${ENTRY} ${MFEM_CXXFLAGS}")
-            endforeach()
-          endif()
-        endif()
-      endif()
-    elseif (MFEM_USE_HIP)
-      set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xhip")
-      foreach(ENTRY IN LISTS CMAKE_HIP_ARCHITECTURES)
-        set(MFEM_CXXFLAGS "--offload-arch=${ENTRY} ${MFEM_CXXFLAGS}")
-      endforeach()
-    endif()
-  endif()
   set(MFEM_TPLFLAGS "")
   foreach(dir ${TPL_INCLUDE_DIRS})
     set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
@@ -987,9 +930,6 @@ function(mfem_export_mk_files)
     set(MFEM_SHARED NO)
     set(MFEM_STATIC YES)
   endif()
-  if (MFEM_USE_CUDA)
-    set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} -lcudart")
-  endif()
   set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}")
   set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}")
   # For the next 4 variables, these are the values for the build-tree version of
@@ -998,15 +938,8 @@ function(mfem_export_mk_files)
   set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk")
   set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}")
-  if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
-    if (MFEM_CUDA_COMPILER_IS_NVCC)
-      set(MFEM_XLINKER "-Xlinker=")
-    else()
-      set(MFEM_XLINKER "${CMAKE_CUDA_LINKER_WRAPPER_FLAG}")
-    endif()
-  else()
-    set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
-  endif()
+  # TODO: CUDA/HIP support:
+  set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
   set(MFEM_MPIEXEC ${MPIEXEC})
   if (NOT MFEM_MPIEXEC)
     set(MFEM_MPIEXEC "mpirun")
@@ -1054,21 +987,16 @@ function(mfem_export_mk_files)
     # handle interfaces (e.g., SCOREC::apf)
     if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
     elseif (TARGET "${lib}")
-      mfem_get_target_options(${lib} CompileOpts2 LinkOpts2)
-      # remove generator expressions
-      string(GENEX_STRIP "${CompileOpts2}" CompileOpts)
-      string(GENEX_STRIP "${LinkOpts2}" LinkOpts)
+      mfem_get_target_options(${lib} CompileOpts LinkOpts)
       # Removing duplicates may lead to issues:
       # list(REMOVE_DUPLICATES CompileOpts)
       # list(REMOVE_DUPLICATES LinkOpts)
-      # message(WARNING "${lib}[LinkOpts]: ${LinkOpts}")      
-      # message(WARNING "${lib}[CompileOpts]: ${CompileOpts}")
-      foreach(LOpt IN LISTS LinkOpts)
-        set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpt}")
-      endforeach()
-      foreach(COpt IN LISTS CompileOpts)
-        set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpt}")
-      endforeach()
+      string(REPLACE ";" " " COpts "${CompileOpts}")
+      string(REPLACE ";" " " LOpts "${LinkOpts}")
+      # message(STATUS "${lib}[COpts]: '${COpts}'")
+      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
+      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
+      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
       # message(FATAL_ERROR "***** interface lib found ... exiting *****")
       # handle static and shared libs
     elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
@@ -1076,7 +1004,7 @@ function(mfem_export_mk_files)
       get_filename_component(fullLibName ${lib} NAME_WE)
       string(REGEX REPLACE "^lib" "" libname ${fullLibName})
       set(MFEM_EXT_LIBS
-        "${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
+          "${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
     else()
       set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${lib}")
     endif()
@@ -1085,7 +1013,7 @@ function(mfem_export_mk_files)
   # Create the build-tree version of 'config.mk'
   configure_file(
     "${PROJECT_SOURCE_DIR}/config/config.mk.in"
-    "${PROJECT_BINARY_DIR}/config/config.mk" @ONLY)
+    "${PROJECT_BINARY_DIR}/config/config.mk")
   # Copy 'test.mk' from the source-tree to the build-tree
   configure_file(
     "${PROJECT_SOURCE_DIR}/config/test.mk"
@@ -1103,7 +1031,7 @@ function(mfem_export_mk_files)
   # Create the install-tree version of 'config.mk'
   configure_file(
     "${PROJECT_SOURCE_DIR}/config/config.mk.in"
-    "${PROJECT_BINARY_DIR}/config/config-install.mk" @ONLY)
+    "${PROJECT_BINARY_DIR}/config/config-install.mk")
 
   # Install rules for 'config.mk' and 'test.mk'
   install(FILES ${PROJECT_SOURCE_DIR}/config/test.mk

config/config.hpp

@@ -62,6 +62,11 @@ constexpr real_t operator""_r(unsigned long long v)
    return static_cast<real_t>(v);
 }
 
+// MFEM bigint type
+
+/// MFEM's "big" integer type.
+typedef long long int bigint;
+
 } // namespace mfem
 
 // Return value for main function in examples that should be skipped by testing

examples/CMakeLists.txt

@@ -215,7 +215,7 @@ if (MFEM_ENABLE_TESTING)
     add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex1p> "-no-vis" "-d" "ceed-cpu" "-pa" "-a"
+      $<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
       ${MPIEXEC_POSTFLAGS})
   endif()
 endif()

examples/amgx/makefile

@@ -64,7 +64,7 @@ PARALLEL_NAME := Parallel AMGX example
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not build)
 
-clean: clean-build clean-exec
+clean: clean-build
 
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)

examples/caliper/makefile

@@ -64,12 +64,12 @@ ex1p-test-par: ex1p
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not built)
 
-clean: clean-build clean-exec
+clean: clean-build clean-exec $(SUBDIRS_CLEAN)
 
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -f refined.mesh mesh.*
-	@rm -f sol.*
+	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*

examples/ex12p.cpp

@@ -5,9 +5,9 @@
 // Sample runs:
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh
-//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 464 -n 10 -o 2 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
-//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 82
+//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3877 -o 2 -sys
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh

examples/ex22p.cpp

@@ -302,21 +302,15 @@ int main(int argc, char *argv[])
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
       sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
       sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
       sol_sock_r << "solution\n" << *pmesh << u_exact->real()
                  << "window_title 'Exact: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
       sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
                  << "window_title 'Exact: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
    }
 
    // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
@@ -540,21 +534,15 @@ int main(int argc, char *argv[])
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
       sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
       sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
       sol_sock_r << "solution\n" << *pmesh << u.real()
                  << "window_title 'Solution: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
       sol_sock_i << "solution\n" << *pmesh << u.imag()
                  << "window_title 'Solution: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
    }
    if (visualization && exact_sol)
    {
@@ -563,21 +551,15 @@ int main(int argc, char *argv[])
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
       sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
       sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
       sol_sock_r << "solution\n" << *pmesh << u_exact->real()
                  << "window_title 'Error: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
       sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
                  << "window_title 'Error: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
    }
    if (visualization)
    {

examples/ex37.cpp

@@ -5,8 +5,8 @@
 // Sample runs:
 //     ex37 -alpha 10
 //     ex37 -alpha 10 -pv
-//     ex37 -lambda 0.1 -mu 0.1 -growth 1
-//     ex37 -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
+//     ex37 -lambda 0.1 -mu 0.1
+//     ex37 -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
 //     ex37 -r 6 -o 1 -alpha 25.0 -epsilon 0.02 -mi 50 -ntol 1e-5
 //
 // Description: This example code demonstrates the use of MFEM to solve a
@@ -55,6 +55,53 @@
 using namespace std;
 using namespace mfem;
 
+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
+ */
+real_t proj(GridFunction &psi, real_t target_volume, real_t tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   LinearForm int_sigmoid_psi(psi.FESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   LinearForm int_der_sigmoid_psi(psi.FESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      const real_t f = int_sigmoid_psi.Sum() - target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      const real_t df = int_der_sigmoid_psi.Sum();
+
+      const real_t dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   return int_sigmoid_psi.Sum();
+}
+
 /*
  * ---------------------------------------------------------------
  *                      ALGORITHM PREAMBLE
@@ -133,11 +180,10 @@ int main(int argc, char *argv[])
    int ref_levels = 5;
    int order = 2;
    real_t alpha = 1.0;
-   real_t growth = 2;
    real_t epsilon = 0.01;
    real_t vol_fraction = 0.5;
    int max_it = 1e3;
-   real_t itol = 1e-2;
+   real_t itol = 1e-1;
    real_t ntol = 1e-4;
    real_t rho_min = 1e-6;
    real_t lambda = 1.0;
@@ -152,8 +198,6 @@ int main(int argc, char *argv[])
                   "Order (degree) of the finite elements.");
    args.AddOption(&alpha, "-alpha", "--alpha-step-length",
                   "Step length for gradient descent.");
-   args.AddOption(&growth, "-growth", "--alpha-growth-rate",
-                  "Growth rate of step length for gradient descent.");
    args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
                   "Length scale for ρ.");
    args.AddOption(&max_it, "-mi", "--max-it",
@@ -288,7 +332,6 @@ int main(int argc, char *argv[])
    }
    FilterSolver->SetEssentialBoundary(ess_bdr_filter);
    FilterSolver->SetupFEM();
-   FilterSolver->AssembleDiffusionBilinear();
 
    BilinearForm mass(&control_fes);
    mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -342,7 +385,7 @@ int main(int argc, char *argv[])
    // 11. Iterate:
    for (int k = 1; k <= max_it; k++)
    {
-      if (k > 1) { alpha = std::pow((real_t) k,growth); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }
 
       mfem::out << "\nStep = " << k << std::endl;
 
@@ -379,9 +422,7 @@ int main(int argc, char *argv[])
 
       // Step 5 - Update design variable ψ ← proj(ψ - αG)
       psi.Add(-alpha, grad);
-      GridFunction alpha_grad(grad);
-      alpha_grad *= alpha;
-      const real_t material_volume = proj(psi, alpha_grad, target_volume);
+      const real_t material_volume = proj(psi, target_volume);
 
       // Compute ||ρ - ρ_old|| in control fes.
       real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);

examples/ex37.hpp

@@ -137,7 +137,7 @@ public:
         exponent(exponent_), rho_min(rho_min_)
    {
       MFEM_ASSERT(rho_min_ >= 0.0, "rho_min must be >= 0");
-      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be < 1");
+      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be > 1");
       MFEM_ASSERT(u, "displacement field is not set");
       MFEM_ASSERT(rho_filter, "density field is not set");
    }
@@ -231,12 +231,9 @@ private:
    FiniteElementCollection * fec = nullptr;
    FiniteElementSpace * fes = nullptr;
    Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
    Array<int> neumann_bdr;
    GridFunction * u = nullptr;
    LinearForm * b = nullptr;
-   BilinearForm * a = nullptr;
-   OperatorPtr A;
    bool parallel;
 #ifdef MFEM_USE_MPI
    ParMesh * pmesh = nullptr;
@@ -270,8 +267,6 @@ public:
    void ResetFEM();
    void SetupFEM();
 
-   void UpdateEssentialTDofs();
-   void AssembleDiffusionBilinear(bool update_ess_tdofs=true);
    void Solve();
    GridFunction * GetFEMSolution();
    LinearForm * GetLinearForm() {return b;}
@@ -376,130 +371,6 @@ public:
 
 };
 
-/**
- * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
- *        ∫_Ω ρ dx = θ vol(Ω) as follows:
- *
- *        1. Compute the root of the R → R function
- *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
- *           using the Illinois method
- *        2. Set ψ ← ψ + c.
- *
- * @param psi a GridFunction to be updated
- * @param alpha_grad alpha multiplied by gradient
- * @param target_volume θ vol(Ω)
- * @param tol Illinois iteration tolerance
- * @param max_its Illinois maximum iteration number
- * @return real_t Final volume (∫_Ω sigmoid(ψ) dx)
- */
-real_t proj(GridFunction &psi, GridFunction &alpha_grad, real_t target_volume,
-            real_t tol = 1e-12, int max_its = 100)
-{
-#ifdef MFEM_USE_MPI
-   FiniteElementSpace *fes = psi.FESpace();
-   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace*>(fes);
-#endif
-   ConstantCoefficient zero_cf(0.0);
-   real_t a = -alpha_grad.ComputeMaxError(zero_cf);
-   real_t b = -a;
-   real_t y = 0.0;
-
-   MappedGridFunctionCoefficient sigmoid_psi(
-   &psi, [&y](const real_t x) { return sigmoid(x + y); });
-   std::unique_ptr<LinearForm> int_sigmoid_psi;
-#ifdef MFEM_USE_MPI
-   ParGridFunction *par_psi = dynamic_cast<ParGridFunction *>(&psi);
-   if (par_psi)
-   {
-      int_sigmoid_psi.reset(new ParLinearForm(par_psi->ParFESpace()));
-   }
-   else
-   {
-      int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
-   }
-#else
-   int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
-#endif
-   int_sigmoid_psi->AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
-
-   y = a;
-   int_sigmoid_psi->Assemble();
-   real_t f_a = int_sigmoid_psi->Sum();  // f_a := f(a) + θ vol(Ω)
-
-   y = b;
-   int_sigmoid_psi->Assemble();
-   real_t f_b = int_sigmoid_psi->Sum();  // f_b := f(b) + θ vol(Ω)
-#ifdef MFEM_USE_MPI
-   if (pfes)
-   {
-      MPI_Allreduce(MPI_IN_PLACE, &f_a, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE, &f_b, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-   }
-#endif
-   f_a -= target_volume; // f_a := f(a)
-   f_b -= target_volume; // f_b := f(b)
-   real_t c = 0.0;
-   real_t f_c = 0.0;
-   int side = 0;
-
-   bool done = false;
-   for (int k=0; k < max_its; k++)
-   {
-      c = (f_a * b - f_b * a) / (f_a - f_b);
-
-      if (abs(b - a) < tol * abs(b + a)) { done = true; break; }
-
-      y = c;
-      int_sigmoid_psi->Assemble();
-      f_c = int_sigmoid_psi->Sum(); // f_c := f(c) + θ vol(Ω)
-#ifdef MFEM_USE_MPI
-      if (pfes)
-      {
-         MPI_Allreduce(MPI_IN_PLACE, &f_c, 1, MPITypeMap<real_t>::mpi_type,
-                       MPI_SUM, MPI_COMM_WORLD);
-      }
-#endif
-      f_c -= target_volume; // f_c := f(c)
-
-      if (f_c * f_b > 0)
-      {
-         b = c;
-         f_b = f_c;
-         if (side == -1) { f_a /= 2.0; }
-         side = -1;
-      }
-      else if (f_c * f_a > 0)
-      {
-         a = c;
-         f_a = f_c;
-         if (side == 1) { f_b /= 2.0; }
-         side = 1;
-      }
-      else
-      {
-         done = true; break;
-      }
-   }
-   if (!done)
-   {
-      mfem_warning("Projection reached maximum iteration without converging. "
-                   "Result may not be accurate.");
-   }
-   y = 0.0;
-   psi += c;
-   int_sigmoid_psi->Assemble();
-   real_t material_volume = int_sigmoid_psi->Sum();
-#ifdef MFEM_USE_MPI
-   if (pfes)
-   {
-      MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
-                    MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
-   }
-#endif
-   return material_volume;
-}
 
 // Poisson solver
 
@@ -551,8 +422,12 @@ void DiffusionSolver::SetupFEM()
    }
 }
 
-void DiffusionSolver::UpdateEssentialTDofs()
+void DiffusionSolver::Solve()
 {
+   OperatorPtr A;
+   Vector B, X;
+   Array<int> ess_tdof_list;
+
 #ifdef MFEM_USE_MPI
    if (parallel)
    {
@@ -565,39 +440,7 @@ void DiffusionSolver::UpdateEssentialTDofs()
 #else
    fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
 #endif
-}
-
-void DiffusionSolver::AssembleDiffusionBilinear(bool update_ess_tdofs)
-{
-   if (update_ess_tdofs)
-   {
-      UpdateEssentialTDofs();
-   }
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      a = new ParBilinearForm(pfes);
-   }
-   else
-   {
-      a = new BilinearForm(fes);
-   }
-#else
-   a = new BilinearForm(fes);
-#endif
-   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
-   if (masscf)
-   {
-      a->AddDomainIntegrator(new MassIntegrator(*masscf));
-   }
-   a->Assemble();
-   a->FormSystemMatrix(ess_tdof_list, A);
-}
-
-void DiffusionSolver::Solve()
-{
-   Vector B, X;
-
+   *u=0.0;
    if (b)
    {
       delete b;
@@ -632,33 +475,31 @@ void DiffusionSolver::Solve()
 
    b->Assemble();
 
-   *u=0.0;
-   if (essbdr_cf)
-   {
-      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
-   }
+   BilinearForm * a = nullptr;
 
 #ifdef MFEM_USE_MPI
    if (parallel)
    {
-      X.SetSize(pfes->TrueVSize());
-      B.SetSize(pfes->TrueVSize());
-      dynamic_cast<ParGridFunction*>(u)->ParallelAssemble(X);
-      dynamic_cast<ParLinearForm*>(b)->ParallelAssemble(B);
-      dynamic_cast<ParBilinearForm*>(a)->ParallelEliminateTDofsInRHS(
-         ess_tdof_list, X, B);
+      a = new ParBilinearForm(pfes);
    }
    else
    {
-      X.NewDataAndSize(u->GetData(), u->Size());
-      B.NewDataAndSize(b->GetData(), b->Size());
-      a->EliminateVDofsInRHS(ess_tdof_list, X, B);
+      a = new BilinearForm(fes);
    }
 #else
-   X.NewDataAndSize(u->GetData(), u->Size());
-   B.NewDataAndSize(b->GetData(), b->Size());
-   a->EliminateVDofsInRHS(ess_tdof_list, X, B);
+   a = new BilinearForm(fes);
 #endif
+   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
+   if (masscf)
+   {
+      a->AddDomainIntegrator(new MassIntegrator(*masscf));
+   }
+   a->Assemble();
+   if (essbdr_cf)
+   {
+      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
+   }
+   a->FormLinearSystem(ess_tdof_list, *u, *b, A, X, B);
 
    CGSolver * cg = nullptr;
    Solver * M = nullptr;
@@ -687,6 +528,7 @@ void DiffusionSolver::Solve()
    delete M;
    delete cg;
    a->RecoverFEMSolution(X, *b, *u);
+   delete a;
 }
 
 GridFunction * DiffusionSolver::GetFEMSolution()
@@ -718,8 +560,6 @@ DiffusionSolver::~DiffusionSolver()
 #endif
    delete fec; fec = nullptr;
    delete b;
-   A.Clear();
-   delete a;
 }
 
 

examples/ex37p.cpp

@@ -4,8 +4,8 @@
 //
 // Sample runs:
 //    mpirun -np 4 ex37p -alpha 10 -pv
-//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1 -growth 1
-//    mpirun -np 4 ex37p -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
+//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1
+//    mpirun -np 4 ex37p -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
 //    mpirun -np 4 ex37p -r 6 -o 2 -alpha 10.0 -epsilon 0.02 -mi 50 -ntol 1e-5
 //
 // Description: This example code demonstrates the use of MFEM to solve a
@@ -54,6 +54,61 @@
 using namespace std;
 using namespace mfem;
 
+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
+ */
+real_t proj(ParGridFunction &psi, real_t target_volume, real_t tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   ParLinearForm int_sigmoid_psi(psi.ParFESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   ParLinearForm int_der_sigmoid_psi(psi.ParFESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      real_t f = int_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
+      f -= target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      real_t df = int_der_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
+
+      const real_t dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   real_t material_volume = int_sigmoid_psi.Sum();
+   MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
+                 MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
+   return material_volume;
+}
+
 /*
  * ---------------------------------------------------------------
  *                      ALGORITHM PREAMBLE
@@ -138,11 +193,10 @@ int main(int argc, char *argv[])
    int ref_levels = 5;
    int order = 2;
    real_t alpha = 1.0;
-   real_t growth = 2;
    real_t epsilon = 0.01;
    real_t vol_fraction = 0.5;
    int max_it = 1e3;
-   real_t itol = 1e-2;
+   real_t itol = 1e-1;
    real_t ntol = 1e-4;
    real_t rho_min = 1e-6;
    real_t lambda = 1.0;
@@ -157,8 +211,6 @@ int main(int argc, char *argv[])
                   "Order (degree) of the finite elements.");
    args.AddOption(&alpha, "-alpha", "--alpha-step-length",
                   "Step length for gradient descent.");
-   args.AddOption(&growth, "-growth", "--alpha-growth-rate",
-                  "Growth rate of step length for gradient descent.");
    args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
                   "Length scale for ρ.");
    args.AddOption(&max_it, "-mi", "--max-it",
@@ -307,7 +359,6 @@ int main(int argc, char *argv[])
    }
    FilterSolver->SetEssentialBoundary(ess_bdr_filter);
    FilterSolver->SetupFEM();
-   FilterSolver->AssembleDiffusionBilinear();
 
    ParBilinearForm mass(&control_fes);
    mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -361,7 +412,7 @@ int main(int argc, char *argv[])
    // 11. Iterate:
    for (int k = 1; k <= max_it; k++)
    {
-      if (k > 1) { alpha = std::pow((real_t) k,growth); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }
 
       if (myid == 0)
       {
@@ -401,9 +452,7 @@ int main(int argc, char *argv[])
 
       // Step 5 - Update design variable ψ ← proj(ψ - αG)
       psi.Add(-alpha, grad);
-      ParGridFunction alpha_grad(grad);
-      alpha_grad *= alpha;
-      const real_t material_volume = proj(psi, alpha_grad, target_volume);
+      const real_t material_volume = proj(psi, target_volume);
 
       // Compute ||ρ - ρ_old|| in control fes.
       real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);

examples/ginkgo/makefile

@@ -76,4 +76,4 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -f refined.mesh sol.gf mesh.* sol.*
+	@rm -f refined.mesh sol.gf

examples/makefile

@@ -71,7 +71,6 @@ endif
 
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
-SUBDIRS_TEST_NOCLEAN = $(addsuffix /test-noclean,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))
 SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
 
@@ -88,9 +87,8 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
 
 all: $(EXAMPLES) $(SUBDIRS_ALL)
 
-.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) \
-   $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
-$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) $(SUBDIRS_CLEAN):
+.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
+$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN):
 	$(MAKE) -C $(@D) $(@F)
 $(SUBDIRS_TPRINT):
 	@$(MAKE) -C $(@D) $(@F)
@@ -109,7 +107,6 @@ endif
 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)
 test: $(SUBDIRS_TEST)
-test-noclean: $(SUBDIRS_TEST_NOCLEAN)
 test-print: $(SUBDIRS_TPRINT)
 
 # Testing: Parallel vs. serial runs
@@ -160,8 +157,6 @@ ex37-test-seq: ex37
 	@$(call mfem-test,$<,, Serial example,-mi 3)
 ex37p-test-par: ex37p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-mi 3)
-ex39-test-seq: ex39
-	@$(call mfem-test,$<,, Serial example,-m ../data/compass.mesh)
 ex41-test-seq: ex41
 	@$(call mfem-test,$<,, Serial example,-tf 1.0)
 ex41p-test-par: ex41p

fem/bilinearform.cpp

@@ -729,8 +729,7 @@ void BilinearForm::Assemble(int skip_zeros)
          tr = mesh -> GetBdrFaceTransformations (i);
          if (tr != NULL)
          {
-            mfem::DofTransformation doftrans;
-            fes -> GetElementVDofs (tr -> Elem1No, vdofs, doftrans);
+            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
             fe1 = fes -> GetFE (tr -> Elem1No);
             // The fe2 object is really a dummy and not used on the boundaries,
             // but we can't dereference a NULL pointer, and we don't want to
@@ -744,7 +743,6 @@ void BilinearForm::Assemble(int skip_zeros)
 
                boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
                                                               elemmat);
-               doftrans.TransformDual(elemmat);
                mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
             }
          }
@@ -1725,7 +1723,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
          }
       }
 
-      DofTransformation dom_dof_trans, ran_dof_trans;
       for (int i = 0; i < trial_fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
@@ -1734,8 +1731,8 @@ void MixedBilinearForm::Assemble(int skip_zeros)
          ftr = mesh -> GetBdrFaceTransformations (i);
          if (ftr != NULL)
          {
-            trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs, dom_dof_trans);
-            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs, ran_dof_trans);
+            trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
             trial_fe1 = trial_fes->GetFE(ftr->Elem1No);
             test_fe1 = test_fes->GetFE(ftr->Elem1No);
             // The test_fe2 object is really a dummy and not used on the
@@ -1751,7 +1748,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
                boundary_face_integs[k]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
                                                            *test_fe2,
                                                            *ftr, elemmat);
-               TransformDual(ran_dof_trans, dom_dof_trans, elemmat);
                mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
             }
          }

fem/bilininteg.hpp

@@ -2710,7 +2710,7 @@ public:
 
 
 /** Integrator for $(-Q u, \nabla v)$ for Nedelec ($u$) and $H^1$ ($v$) elements.
-    This is equivalent to a weak divergence of the $H(curl)$ basis functions. */
+    This is equivalent to a weak divergence of the $H(curl$ basis functions. */
 class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
 {
 protected:

fem/bounds.cpp

@@ -41,14 +41,9 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
    tol = tol_i;
    lbound.SetSize(ncp, nb);
    ubound.SetSize(ncp, nb);
-   lbound_t.SetSize(nb, ncp);
-   ubound_t.SetSize(nb, ncp);
    nodes.SetSize(nb);
    weights.SetSize(nb);
    control_points.SetSize(ncp);
-   xhat.SetSize(nb);
-   what.SetSize(nb);
-   cphat.SetSize(ncp);
 
    auto scalenodes = [](const Vector &in, const real_t a, const real_t b) -> Vector
    {
@@ -95,10 +90,6 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
       MFEM_ABORT("Unsupported interval points. Use [0,1].\n");
    }
    control_points = scalenodes(control_points, 0.0, 1.0); // rescale to [0,1]
-   for (int i = 0; i < ncp; i++)
-   {
-      cphat(i) = 2.0*control_points(i) - 1.0;
-   }
 
    Poly_1D::Basis &basis1d(poly1d.GetBasis(nb-1, b_type));
 
@@ -154,8 +145,6 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
                lbound(j,i) = std::max(lbound(j,i),0_r);
             }
          }
-         lbound_t(i,j) = lbound(j,i);
-         ubound_t(i,j) = ubound(j,i);
       }
    }
 
@@ -187,11 +176,6 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
          nodes(i) = irule.IntPoint(i).x;
       }
    }
-   for (int i = 0; i < nb; i++)
-   {
-      xhat(i) = 2.0*nodes(i) - 1.0;
-      what(i) = 2.0*weights(i);
-   }
 
    if (b_type == 2)
    {
@@ -771,4 +755,4 @@ void PLBound::Print(std::ostream &outp) const
    ubound.Print(outp);
 }
 
-}
+}

fem/bounds.hpp

@@ -13,7 +13,6 @@
 #define MFEM_BOUNDS
 
 #include "../config/config.hpp"
-#include "../general/forall.hpp"
 #include "fespace.hpp"
 
 namespace mfem
@@ -61,9 +60,7 @@ private:
    bool proj = true; // Use linear projection to compute bounds.
    real_t tol = 0.0; // offset bounds to avoid round-off errors
    Vector nodes, weights, control_points;
-   Vector xhat, what, cphat;
    DenseMatrix lbound, ubound; // ncp x nb matrices with bounds of all bases
-   DenseMatrix lbound_t, ubound_t; // nb x ncp transposes for device kernel
    // Some auxillary storage for computing the bounds with Bernstein
    DenseMatrix basisMatNodes; // Bernstein bases at equispaced nodes
    DenseMatrix basisMatInt;   // Bernstein bases at GLL nodes
@@ -116,10 +113,7 @@ public:
     *  @details This projection increases the computational cost but results in
     *  tighter bounds.
     */
-   void SetProjectionFlagForBounding(bool proj_)
-   {
-      proj = proj_;
-   }
+   void SetProjectionFlagForBounding(bool proj_) { proj = proj_; }
 
    /** @brief Compute piecewise linear bounds for the lexicographically-ordered
     *  nodal coefficients in @a coeff in 1D/2D/3D.
@@ -143,23 +137,9 @@ public:
    /// Get number of control points used to compute the bounds.
    int GetNControlPoints() const { return ncp; }
 
-   /// Get the underlying 1D basis type.
-   int GetBasisType() const { return b_type; }
-
    /// Get 1D control point locations (lexicographic order) in [0,1].
    const Vector &GetControlPoints() const { return control_points; }
 
-   /** @brief Compute element-wise bounds from a lexicographic E-vector.
-    *
-    *  @details The expected layout of @a e_vec is `ND x VDIM x NE`, where
-    *  `ND = nb^rdim`, `VDIM = fes_vdim`, and `NE` is the number of elements.
-    *  The output layout matches GridFunction::GetElementBounds:
-    *  `NE x active_vdim`, with the element index varying fastest.
-    */
-   void GetElementBoundsKernel(const int rdim, const int fes_vdim,
-                               const Vector &e_vec, Vector &lower,
-                               Vector &upper, const int vdim = 0) const;
-
    /** @brief Get lower and upper bounding matrix (ncp^dim x nb^dim)
     *
     *  @details The matrices can be used to compute the bounds at control points
@@ -203,600 +183,6 @@ private:
               const int cp_type_i, const real_t tol_i);
 };
 
-namespace internal
-{
-
-struct PLBoundDeviceData
-{
-   int nb;
-   int ncp;
-   const real_t *xhat;
-   const real_t *what;
-   const real_t *cphat;
-   const real_t *lbound;
-   const real_t *ubound;
-};
-
-template<int T_NB = 0, bool T_PROJ = true>
-inline void GetElementBoundsKernel1D(const PLBoundDeviceData &data,
-                                     const int fes_vdim,
-                                     const int ne,
-                                     const Vector &e_vec,
-                                     Vector &lower,
-                                     Vector &upper,
-                                     const int comp0,
-                                     const int ncomp)
-{
-   constexpr int GENERIC_MAX_ND = 32;
-   constexpr int MAX_ND = T_NB ? T_NB : GENERIC_MAX_ND;
-   constexpr int BLOCK_X = 2*MAX_ND;
-
-   const int nd = T_NB ? T_NB : data.nb;
-   MFEM_VERIFY(nd <= MAX_ND,
-               "Device element bounds kernel supports up to 32 "
-               "1D degrees of freedom.");
-
-   const auto E = Reshape(e_vec.Read(), nd, fes_vdim, ne);
-   auto L = Reshape(lower.Write(), ne, ncomp);
-   auto U = Reshape(upper.Write(), ne, ncomp);
-
-   mfem::forall_2D<BLOCK_X>(ne*ncomp, BLOCK_X, 1,
-                            [=] MFEM_HOST_DEVICE (int ec)
-   {
-      const int e = ec % ne;
-      const int c = ec / ne;
-      const int vc = comp0 + c;
-      const real_t *coeff = &E(0, vc, e);
-      const int tid = MFEM_THREAD_ID(x);
-
-      MFEM_SHARED real_t sproj[MAX_ND];
-      MFEM_SHARED real_t ssum0[MAX_ND];
-      MFEM_SHARED real_t ssum1[MAX_ND];
-      MFEM_SHARED real_t smin[BLOCK_X];
-      MFEM_SHARED real_t smax[BLOCK_X];
-      MFEM_SHARED real_t sa0;
-      MFEM_SHARED real_t sa1;
-
-      MFEM_FOREACH_THREAD(i, x, nd)
-      {
-         if constexpr (T_PROJ)
-         {
-            const real_t x = data.xhat[i];
-            const real_t w = data.what[i];
-            ssum0[i] = 0.5*coeff[i]*w;
-            ssum1[i] = 1.5*coeff[i]*w*x;
-         }
-         else
-         {
-            ssum0[i] = 0.0;
-            ssum1[i] = 0.0;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(ii, x, 1)
-      {
-         sa0 = 0.0;
-         sa1 = 0.0;
-         for (int i = 0; i < nd; i++)
-         {
-            sa0 += ssum0[i];
-            sa1 += ssum1[i];
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(i, x, nd)
-      {
-         if constexpr (T_PROJ)
-         {
-            const real_t x = data.xhat[i];
-            sproj[i] = coeff[i] - sa0 - sa1*x;
-         }
-         else
-         {
-            sproj[i] = coeff[i];
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      real_t lower_local = HUGE_VAL;
-      real_t upper_local = -HUGE_VAL;
-      MFEM_FOREACH_THREAD(j, x, data.ncp)
-      {
-         real_t lo = 0.0;
-         real_t hi = 0.0;
-         if constexpr (T_PROJ)
-         {
-            const real_t xcp = data.cphat[j];
-            lo = sa0 + sa1*xcp;
-            hi = lo;
-         }
-
-         for (int i = 0; i < nd; i++)
-         {
-            const real_t val = sproj[i];
-            const real_t lv = data.lbound[j + i*data.ncp]*val;
-            const real_t uv = data.ubound[j + i*data.ncp]*val;
-            lo += lv < uv ? lv : uv;
-            hi += lv > uv ? lv : uv;
-         }
-         lower_local = lower_local < lo ? lower_local : lo;
-         upper_local = upper_local > hi ? upper_local : hi;
-      }
-
-      smin[tid] = lower_local;
-      smax[tid] = upper_local;
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(ii, x, 1)
-      {
-         real_t lower_ec = smin[0];
-         real_t upper_ec = smax[0];
-         const int nthreads = MFEM_THREAD_SIZE(x);
-         const int nactive = data.ncp < nthreads ? data.ncp : nthreads;
-         for (int t = 1; t < nactive; t++)
-         {
-            lower_ec = lower_ec < smin[t] ? lower_ec : smin[t];
-            upper_ec = upper_ec > smax[t] ? upper_ec : smax[t];
-         }
-         L(e, c) = lower_ec;
-         U(e, c) = upper_ec;
-      }
-   });
-}
-
-template<int T_NB = 0, int T_NCP = 0, bool T_PROJ = true>
-inline void GetElementBoundsKernel2D(const PLBoundDeviceData &data,
-                                     const int fes_vdim,
-                                     const int ne,
-                                     const Vector &e_vec,
-                                     Vector &lower,
-                                     Vector &upper,
-                                     const int comp0,
-                                     const int ncomp)
-{
-   constexpr int DEFAULT_MAX_NB = 8;
-   constexpr int DEFAULT_MAX_CP = 3*DEFAULT_MAX_NB;
-   constexpr int MAX_NB = T_NB ? T_NB : DEFAULT_MAX_NB;
-   constexpr int MAX_CP = T_NCP ? T_NCP : DEFAULT_MAX_CP;
-   constexpr int MAX_THREADS = MAX_CP*MAX_CP;
-
-   const int nb = data.nb;
-   const int ncp = data.ncp;
-   const int nd = nb*nb;
-   MFEM_VERIFY(nb <= MAX_NB,
-               "Device 2D element bounds kernel exceeds its compile-time "
-               "1D degree bound.");
-   MFEM_VERIFY(ncp <= MAX_CP,
-               "Device 2D element bounds kernel exceeds its compile-time "
-               "control-point bound.");
-   MFEM_VERIFY(ncp*ncp <= MAX_THREADS,
-               "Device 2D element bounds kernel exceeds its compile-time "
-               "thread-block bound.");
-
-   const auto E = Reshape(e_vec.Read(), nd, fes_vdim, ne);
-   auto L = Reshape(lower.Write(), ne, ncomp);
-   auto U = Reshape(upper.Write(), ne, ncomp);
-
-   mfem::forall_2D<MAX_THREADS>(ne*ncomp, ncp, ncp,
-                                [=] MFEM_HOST_DEVICE (int ec)
-   {
-      const int e = ec % ne;
-      const int c = ec / ne;
-      const int vc = comp0 + c;
-      const real_t *coeff = &E(0, vc, e);
-      const int tx = MFEM_THREAD_ID(x);
-      const int ty = MFEM_THREAD_ID(y);
-
-      MFEM_SHARED real_t sproj[MAX_NB*MAX_NB];
-      MFEM_SHARED real_t srow_min[MAX_NB*MAX_CP];
-      MFEM_SHARED real_t srow_max[MAX_NB*MAX_CP];
-      MFEM_SHARED real_t srow_a0[MAX_NB];
-      MFEM_SHARED real_t srow_a1[MAX_NB];
-      MFEM_SHARED real_t sa0[MAX_CP];
-      MFEM_SHARED real_t sa1[MAX_CP];
-      MFEM_SHARED real_t smin[MAX_THREADS];
-      MFEM_SHARED real_t smax[MAX_THREADS];
-
-      // Stage 1a: for each nodal row, form the per-node contributions to the
-      // row-wise linear fit used by the first 1D bounding solve.
-      MFEM_FOREACH_THREAD(jrow, y, nb)
-      {
-         const real_t *row_coeff = coeff + jrow*nb;
-         const int row_ncp_off = jrow*MAX_CP;
-         MFEM_FOREACH_THREAD(i, x, nb)
-         {
-            if constexpr (T_PROJ)
-            {
-               const real_t x = data.xhat[i];
-               const real_t w = data.what[i];
-               srow_min[row_ncp_off + i] = 0.5*row_coeff[i]*w;
-               srow_max[row_ncp_off + i] = 1.5*row_coeff[i]*w*x;
-            }
-            else
-            {
-               srow_min[row_ncp_off + i] = 0.0;
-               srow_max[row_ncp_off + i] = 0.0;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 1b: reduce the row-wise projection coefficients a0/a1.
-      if constexpr (T_PROJ)
-      {
-         MFEM_FOREACH_THREAD(jrow, y, nb)
-         {
-            const int row_ncp_off = jrow*MAX_CP;
-            real_t a0 = 0.0;
-            real_t a1 = 0.0;
-            MFEM_FOREACH_THREAD(ii, x, 1)
-            {
-               for (int i = 0; i < nb; i++)
-               {
-                  a0 += srow_min[row_ncp_off + i];
-                  a1 += srow_max[row_ncp_off + i];
-               }
-               srow_a0[jrow] = a0;
-               srow_a1[jrow] = a1;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      // Stage 1c: subtract the row-wise linear fit once and cache the
-      // projected row coefficients for reuse across all x-control points.
-      MFEM_FOREACH_THREAD(jrow, y, nb)
-      {
-         const real_t *row_coeff = coeff + jrow*nb;
-         MFEM_FOREACH_THREAD(i, x, nb)
-         {
-            if constexpr (T_PROJ)
-            {
-               const real_t x = data.xhat[i];
-               sproj[jrow*MAX_NB + i] = row_coeff[i]
-                                        - srow_a0[jrow] - srow_a1[jrow]*x;
-            }
-            else
-            {
-               sproj[jrow*MAX_NB + i] = row_coeff[i];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 1d: solve the first 1D bounding problem along each nodal row and
-      // store bounds at every x-direction control point.
-      MFEM_FOREACH_THREAD(icp, x, ncp)
-      {
-         MFEM_FOREACH_THREAD(jrow, y, nb)
-         {
-            const int row_cp_off = jrow*ncp;
-            real_t lo = 0.0;
-            real_t hi = 0.0;
-            if constexpr (T_PROJ)
-            {
-               const real_t xcp = data.cphat[icp];
-               lo = srow_a0[jrow] + srow_a1[jrow]*xcp;
-               hi = lo;
-            }
-            for (int i = 0; i < nb; i++)
-            {
-               const real_t val = sproj[jrow*MAX_NB + i];
-               const real_t lv = data.lbound[icp + i*data.ncp]*val;
-               const real_t uv = data.ubound[icp + i*data.ncp]*val;
-               lo += lv < uv ? lv : uv;
-               hi += lv > uv ? lv : uv;
-            }
-            srow_min[row_cp_off + icp] = lo;
-            srow_max[row_cp_off + icp] = hi;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 2a: from the row bounds, form the per-row contributions to the
-      // second 1D projection solve in the y-direction.
-      MFEM_FOREACH_THREAD(icp, x, ncp)
-      {
-         MFEM_FOREACH_THREAD(jrow, y, nb)
-         {
-            const int row_cp_off = jrow*ncp;
-            if constexpr (T_PROJ)
-            {
-               const real_t x = data.xhat[jrow];
-               const real_t w = data.what[jrow];
-               const real_t t = 0.5*(srow_min[row_cp_off + icp] +
-                                     srow_max[row_cp_off + icp]);
-               smin[row_cp_off + icp] = 0.5*t*w;
-               smax[row_cp_off + icp] = 1.5*t*w*x;
-            }
-            else
-            {
-               smin[row_cp_off + icp] = 0.0;
-               smax[row_cp_off + icp] = 0.0;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 2b: reduce the y-direction projection coefficients for each
-      // x-control-point column.
-      MFEM_FOREACH_THREAD(jj, y, 1)
-      {
-         MFEM_FOREACH_THREAD(icp, x, ncp)
-         {
-            real_t a0 = 0.0;
-            real_t a1 = 0.0;
-            for (int jrow = 0; jrow < nb; jrow++)
-            {
-               a0 += smin[jrow*ncp + icp];
-               a1 += smax[jrow*ncp + icp];
-            }
-            sa0[icp] = a0;
-            sa1[icp] = a1;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 2c: subtract the y-direction linear fit from the intermediate
-      // row bounds so the final tensor-product bound uses the perturbation.
-      if constexpr (T_PROJ)
-      {
-         MFEM_FOREACH_THREAD(icp, x, ncp)
-         {
-            MFEM_FOREACH_THREAD(jrow, y, nb)
-            {
-               const int row_cp_off = jrow*ncp;
-               const real_t x = data.xhat[jrow];
-               const real_t t = sa0[icp] + sa1[icp]*x;
-               srow_min[row_cp_off + icp] -= t;
-               srow_max[row_cp_off + icp] -= t;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 3: each thread now owns one 2D control point (icp, kcp) and
-      // accumulates its final lower/upper bound from the row-bound data.
-      MFEM_FOREACH_THREAD(icp, x, ncp)
-      {
-         MFEM_FOREACH_THREAD(kcp, y, ncp)
-         {
-            real_t lo = 0.0;
-            real_t hi = 0.0;
-            if constexpr (T_PROJ)
-            {
-               const real_t xcp = data.cphat[kcp];
-               lo = sa0[icp] + sa1[icp]*xcp;
-               hi = lo;
-            }
-            for (int jrow = 0; jrow < nb; jrow++)
-            {
-               const real_t w0 = srow_min[jrow*ncp + icp];
-               const real_t w1 = srow_max[jrow*ncp + icp];
-               const real_t lb = data.lbound[kcp + jrow*data.ncp];
-               const real_t ub = data.ubound[kcp + jrow*data.ncp];
-               const real_t v0 = lb*w0;
-               const real_t v1 = ub*w0;
-               const real_t v2 = lb*w1;
-               const real_t v3 = ub*w1;
-               real_t vlo = v0 < v1 ? v0 : v1;
-               real_t vhi = v0 > v1 ? v0 : v1;
-               vlo = vlo < v2 ? vlo : v2;
-               vlo = vlo < v3 ? vlo : v3;
-               vhi = vhi > v2 ? vhi : v2;
-               vhi = vhi > v3 ? vhi : v3;
-               lo += vlo;
-               hi += vhi;
-            }
-            const int slot = kcp*ncp + icp;
-            smin[slot] = lo;
-            smax[slot] = hi;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      const int lane = ty*ncp + tx;
-      const int nactive = ncp*ncp;
-      const int nthreads = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
-
-      // Reduce all 2D control-point bounds to one lower/upper pair per
-      // (element, component).
-      if (nthreads == 1)
-      {
-         if (tx == 0 && ty == 0)
-         {
-            real_t lower_ec = smin[0];
-            real_t upper_ec = smax[0];
-            for (int t = 1; t < nactive; t++)
-            {
-               lower_ec = lower_ec < smin[t] ? lower_ec : smin[t];
-               upper_ec = upper_ec > smax[t] ? upper_ec : smax[t];
-            }
-            L(e, c) = lower_ec;
-            U(e, c) = upper_ec;
-         }
-      }
-      else
-      {
-         for (int stride = (nactive + 1)/2; stride > 0;
-              stride = (stride + 1)/2)
-         {
-            if (lane < stride && lane + stride < nactive)
-            {
-               smin[lane] = smin[lane] < smin[lane + stride] ?
-                            smin[lane] : smin[lane + stride];
-               smax[lane] = smax[lane] > smax[lane + stride] ?
-                            smax[lane] : smax[lane + stride];
-            }
-            MFEM_SYNC_THREAD;
-            if (stride == 1) { break; }
-         }
-
-         if (lane == 0)
-         {
-            L(e, c) = smin[0];
-            U(e, c) = smax[0];
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-inline void PLBound::GetElementBoundsKernel(const int rdim, const int fes_vdim,
-                                            const Vector &e_vec,
-                                            Vector &lower, Vector &upper,
-                                            const int vdim) const
-{
-   MFEM_VERIFY(b_type != BasisType::Positive,
-               "Bernstein device bounds are not implemented.");
-   if (rdim == 3)
-   {
-      MFEM_ABORT("Device element bounds kernel currently only supports 1D/2D.");
-   }
-   MFEM_VERIFY(rdim == 1 || rdim == 2, "Invalid element dimension.");
-   MFEM_VERIFY(vdim >= -1 && vdim <= fes_vdim, "Invalid vector component.");
-   const int nd = static_cast<int>(std::pow(nb, rdim));
-   const int ne = e_vec.Size()/(nd*fes_vdim);
-   const int ncomp = (vdim > 0) ? 1 : fes_vdim;
-
-   lower.SetSize(ne*ncomp, e_vec);
-   upper.SetSize(ne*ncomp, e_vec);
-   lower.UseDevice(true);
-   upper.UseDevice(true);
-
-   if (!proj)
-   {
-      MFEM_ABORT("Device element bounds kernel currently requires projection "
-                 "enabled.");
-   }
-
-   const real_t *dxhat = xhat.Read();
-   const real_t *dwhat = what.Read();
-   const real_t *dcphat = cphat.Read();
-   const real_t *dlbound = lbound.Read();
-   const real_t *dubound = ubound.Read();
-
-   internal::PLBoundDeviceData data
-   {
-      nb,
-      ncp,
-      dxhat,
-      dwhat,
-      dcphat,
-      dlbound,
-      dubound
-   };
-
-   const int comp0 = (vdim > 0) ? (vdim - 1) : 0;
-
-   if (rdim == 1)
-   {
-      switch (nb)
-      {
-         case 2: return internal::GetElementBoundsKernel1D<2, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 3: return internal::GetElementBoundsKernel1D<3, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 4: return internal::GetElementBoundsKernel1D<4, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 5: return internal::GetElementBoundsKernel1D<5, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 6: return internal::GetElementBoundsKernel1D<6, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 7: return internal::GetElementBoundsKernel1D<7, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 8: return internal::GetElementBoundsKernel1D<8, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 9: return internal::GetElementBoundsKernel1D<9, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 10: return internal::GetElementBoundsKernel1D<10, true>(data, fes_vdim, ne,
-                                                                       e_vec, lower, upper,
-                                                                       comp0, ncomp);
-         default: return internal::GetElementBoundsKernel1D<0, true>(data, fes_vdim, ne,
-                                                                      e_vec, lower, upper,
-                                                                      comp0, ncomp);
-      }
-   }
-#define MFEM_PLBOUND_2D_DISPATCH(NB, NCP) \
-   return internal::GetElementBoundsKernel2D<NB, NCP, true>(data, fes_vdim, ne, \
-                                                            e_vec, lower, upper, \
-                                                            comp0, ncomp)
-   switch (nb)
-   {
-      case 2:
-         switch (ncp)
-         {
-            case 4: MFEM_PLBOUND_2D_DISPATCH(2, 4);
-            case 6: MFEM_PLBOUND_2D_DISPATCH(2, 6);
-            case 8: MFEM_PLBOUND_2D_DISPATCH(2, 8);
-         }
-         break;
-      case 3:
-         switch (ncp)
-         {
-            case 6: MFEM_PLBOUND_2D_DISPATCH(3, 6);
-            case 9: MFEM_PLBOUND_2D_DISPATCH(3, 9);
-            case 12: MFEM_PLBOUND_2D_DISPATCH(3, 12);
-         }
-         break;
-      case 4:
-         switch (ncp)
-         {
-            case 8: MFEM_PLBOUND_2D_DISPATCH(4, 8);
-            case 12: MFEM_PLBOUND_2D_DISPATCH(4, 12);
-            case 16: MFEM_PLBOUND_2D_DISPATCH(4, 16);
-         }
-         break;
-      case 5:
-         switch (ncp)
-         {
-            case 10: MFEM_PLBOUND_2D_DISPATCH(5, 10);
-            case 15: MFEM_PLBOUND_2D_DISPATCH(5, 15);
-            case 20: MFEM_PLBOUND_2D_DISPATCH(5, 20);
-         }
-         break;
-      case 6:
-         switch (ncp)
-         {
-            case 12: MFEM_PLBOUND_2D_DISPATCH(6, 12);
-            case 18: MFEM_PLBOUND_2D_DISPATCH(6, 18);
-            case 24: MFEM_PLBOUND_2D_DISPATCH(6, 24);
-         }
-         break;
-      case 7:
-         switch (ncp)
-         {
-            case 14: MFEM_PLBOUND_2D_DISPATCH(7, 14);
-            case 21: MFEM_PLBOUND_2D_DISPATCH(7, 21);
-            case 28: MFEM_PLBOUND_2D_DISPATCH(7, 28);
-         }
-         break;
-      case 8:
-         switch (ncp)
-         {
-            case 16: MFEM_PLBOUND_2D_DISPATCH(8, 16);
-            case 24: MFEM_PLBOUND_2D_DISPATCH(8, 24);
-            case 32: MFEM_PLBOUND_2D_DISPATCH(8, 32);
-         }
-         break;
-   }
-#undef MFEM_PLBOUND_2D_DISPATCH
-   return internal::GetElementBoundsKernel2D<0, 0, true>(data, fes_vdim, ne,
-                                                         e_vec, lower, upper,
-                                                         comp0, ncomp);
-}
-
 } // namespace mfem
 
 #endif // MFEM_BOUNDS

fem/coefficient.hpp

@@ -52,9 +52,6 @@ public:
    /// Get the time for time dependent coefficients
    real_t GetTime() { return time; }
 
-   /// Returns dimension of the vector.
-   int GetVDim() { return 1; }
-
    /** @brief Evaluate the coefficient in the element described by @a T at the
        point @a ip. */
    /** @note When this method is called, the caller must make sure that the

fem/complex_fem.hpp

@@ -82,25 +82,6 @@ public:
    /// underlying #fes
    int VectorDim() const;
 
-   /// Copy assignment. Only the data of the base class Vector is copied.
-   /** It is assumed that this object and @a rhs use FiniteElementSpace%s that
-       have the same size.
-
-       @note Defining this method overwrites the implicitly defined copy
-       assignment operator. */
-   ComplexGridFunction &operator=(const ComplexGridFunction &rhs)
-   { return operator=((const Vector &)rhs); }
-
-   /// Copy the data from @a v.
-   /** The size of @a v must be equal to double of the size of the associated
-       FiniteElementSpace #fes. */
-   ComplexGridFunction &operator=(const Vector &v)
-   {
-      MFEM_ASSERT(fes && v.Size() == 2*fes->GetVSize(), "");
-      Vector::operator=(v);
-      return *this;
-   }
-
    /// Assign constant values to the ComplexGridFunction data.
    ComplexGridFunction &operator=(const std::complex<real_t> & value)
    { *gfr = value.real(); *gfi = value.imag(); return *this; }

fem/datacollection.cpp

@@ -492,8 +492,6 @@ void VisItDataCollection::SaveRootFile()
                            to_padded_string(cycle, pad_digits_cycle) +
                            ".mfem_root";
    std::ofstream root_file(root_name);
-   MFEM_VERIFY(root_file.is_open(),
-               "Failed to open ofstream " << root_name);
    root_file << GetVisItRootString();
    if (!root_file)
    {
@@ -979,10 +977,7 @@ void ParaViewDataCollection::Save()
    // Save the local part of the mesh and grid functions fields to the local
    // VTU file. Also save coefficient fields.
    {
-      std::string os_str = vtu_prefix + GenerateVTUFileName("proc", myid);
-      std::ofstream os(os_str);
-      MFEM_VERIFY(os.is_open(),
-                  "Failed to open ofstream " << os_str);
+      std::ofstream os(vtu_prefix + GenerateVTUFileName("proc", myid));
       os.precision(precision);
       SaveDataVTU(os, levels_of_detail);
    }
@@ -994,10 +989,7 @@ void ParaViewDataCollection::Save()
                   "QuadratureFunction output is not supported for "
                   "ParaViewDataCollection on domain boundary!");
       const std::string &field_name = qfield.first;
-      std::string os_str = vtu_prefix + GenerateVTUFileName(field_name, myid);
-      std::ofstream os(os_str);
-      MFEM_VERIFY(os.is_open(),
-                  "Failed to open ofstream " << os_str);
+      std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
       qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel(), field_name);
    }
 
@@ -1008,10 +1000,7 @@ void ParaViewDataCollection::Save()
    {
       // Create the main PVTU file
       {
-         std::string os_str = vtu_prefix + GeneratePVTUFileName("data");
-         std::ofstream pvtu_out(os_str);
-         MFEM_VERIFY(pvtu_out.is_open(),
-                     "Failed to open ofstream " << os_str);
+         std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data"));
          WritePVTUHeader(pvtu_out);
 
          // Grid function fields and coefficient fields
@@ -1066,10 +1055,8 @@ void ParaViewDataCollection::Save()
          const std::string &q_field_name = q_field.first;
          std::string q_fname = GeneratePVTUPath() + "/"
                                + GeneratePVTUFileName(q_field_name);
-         std::string os_str = col_path + "/" + q_fname;
-         std::ofstream pvtu_out(os_str);
-         MFEM_VERIFY(pvtu_out.is_open(),
-                     "Failed to open ofstream " << os_str);
+
+         std::ofstream pvtu_out(col_path + "/" + q_fname);
          WritePVTUHeader(pvtu_out);
          int vec_dim = q_field.second->GetVDim();
          pvtu_out << "<PPointData>\n";

fem/dfem/integrate.hpp

@@ -90,8 +90,8 @@ void map_quadrature_data_to_fields_impl(
    }
    else
    {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor");
+      MFEM_ABORT("quadrature data mapping to field is not implemented for"
+                 " this field descriptor");
    }
 }
 
@@ -169,9 +169,8 @@ void map_quadrature_data_to_fields_tensor_impl_1d(
    }
    else
    {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        "for this field descriptor with sum factorization on"
-                        " tensor product elements");
+      MFEM_ABORT("quadrature data mapping to field is not implemented for"
+                 " this field descriptor with sum factorization on tensor product elements");
    }
 }
 
@@ -307,9 +306,8 @@ void map_quadrature_data_to_fields_tensor_impl_2d(
    }
    else
    {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor with sum factorization on"
-                        " tensor product elements");
+      MFEM_ABORT("quadrature data mapping to field is not implemented for"
+                 " this field descriptor with sum factorization on tensor product elements");
    }
 }
 
@@ -494,9 +492,8 @@ void map_quadrature_data_to_fields_tensor_impl_3d(
    }
    else
    {
-      MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
-                        " for this field descriptor with sum factorization on"
-                        " tensor product elements");
+      MFEM_ABORT("quadrature data mapping to field is not implemented for"
+                 " this field descriptor with sum factorization on tensor product elements");
    }
 }
 

fem/dgmassinv_kernels.hpp

@@ -57,7 +57,7 @@ void DGMassApply(const int e,
       }
       else if (DIM == 3)
       {
-         SmemPAMassApply3D_Element<TD1D,TQ1D,NBZ,ACCUM>(e, NE, B, pa_data, x, y);
+         SmemPAMassApply3D_Element<TD1D,TQ1D,ACCUM>(e, NE, B, pa_data, x, y);
       }
       else
       {

fem/estimators.hpp

@@ -320,8 +320,8 @@ public:
     error estimation procedure where the flux averaging is replaced by a global
     L2 projection (requiring a mass matrix solve).
 
-    The required BilinearFormIntegrator must implement the method
-    ComputeElementFlux().
+    The required BilinearFormIntegrator must implement the methods
+    ComputeElementFlux() and ComputeFluxEnergy().
 
     Implemented for the parallel case only.
  */
@@ -357,8 +357,8 @@ protected:
 
 public:
    /** @brief Construct a new L2ZienkiewiczZhuEstimator object.
-       @param integ    This BilinearFormIntegrator must implement the method
-                       ComputeElementFlux().
+       @param integ    This BilinearFormIntegrator must implement the methods
+                       ComputeElementFlux() and ComputeFluxEnergy().
        @param sol      The solution field whose error is to be estimated.
        @param flux_fes The L2ZienkiewiczZhuEstimator assumes ownership of this
                        FiniteElementSpace and will call its Update() method when
@@ -382,8 +382,8 @@ public:
    { }
 
    /** @brief Construct a new L2ZienkiewiczZhuEstimator object.
-       @param integ    This BilinearFormIntegrator must implement the method
-                       ComputeElementFlux().
+       @param integ    This BilinearFormIntegrator must implement the methods
+                       ComputeElementFlux() and ComputeFluxEnergy().
        @param sol      The solution field whose error is to be estimated.
        @param flux_fes The L2ZienkiewiczZhuEstimator does NOT assume ownership
                        of this FiniteElementSpace; will call its Update() method

fem/fe/fe_base.cpp

@@ -1044,50 +1044,9 @@ void VectorFiniteElement::SetDerivMembers()
    switch (map_type)
    {
       case H_DIV:
-         switch (dim)
-         {
-            case 3: // div: 3D H_DIV -> 3D INTEGRAL
-               deriv_type = DIV;
-               deriv_range_type = SCALAR;
-               deriv_map_type = INTEGRAL;
-               break;
-            case 2: // div: 2D H_DIV -> 2D INTEGRAL
-               deriv_type = DIV;
-               deriv_range_type = SCALAR;
-               deriv_map_type = INTEGRAL;
-               break;
-            default:
-               MFEM_ABORT("Invalid dimension, Dim = " << dim);
-         }
-         break;
-      case H_DIV_R2D:
-         switch (dim)
-         {
-            case 2: // div: 2D H_DIV_R2D -> 2D INTEGRAL
-               deriv_type = DIV;
-               deriv_range_type = SCALAR;
-               deriv_map_type = INTEGRAL;
-               break;
-            case 1: // div: 1D H_DIV_R2D -> 1D INTEGRAL
-               deriv_type = DIV;
-               deriv_range_type = SCALAR;
-               deriv_map_type = INTEGRAL;
-               break;
-            default:
-               MFEM_ABORT("Invalid dimension, Dim = " << dim);
-         }
-         break;
-      case H_DIV_R1D:
-         switch (dim)
-         {
-            case 1: // div: 1D H_DIV_R1D -> 1D INTEGRAL
-               deriv_type = DIV;
-               deriv_range_type = SCALAR;
-               deriv_map_type = INTEGRAL;
-               break;
-            default:
-               MFEM_ABORT("Invalid dimension, Dim = " << dim);
-         }
+         deriv_type = DIV;
+         deriv_range_type = SCALAR;
+         deriv_map_type = INTEGRAL;
          break;
       case H_CURL:
          switch (dim)
@@ -1105,49 +1064,13 @@ void VectorFiniteElement::SetDerivMembers()
                break;
             case 1:
                deriv_type = NONE;
-               deriv_range_type = UNKNOWN_RANGE_TYPE;
-               deriv_map_type = UNKNOWN_MAP_TYPE;
+               deriv_range_type = SCALAR;
+               deriv_map_type = INTEGRAL;
                break;
             default:
                MFEM_ABORT("Invalid dimension, Dim = " << dim);
          }
          break;
-      case H_CURL_R2D:
-         switch (dim)
-         {
-            case 2:
-               // curl: 2D H_CURL_R2D -> H_DIV_R2D
-               deriv_type = CURL;
-               deriv_range_type = VECTOR;
-               deriv_map_type = H_DIV_R2D;
-               break;
-            case 1:
-               // curl: 1D H_CURL_R2D -> H_DIV_R2D
-               deriv_type = CURL;
-               deriv_range_type = VECTOR;
-               deriv_map_type = H_DIV_R2D;
-               break;
-            default:
-               MFEM_ABORT("Invalid dimension, Dim = " << dim);
-         }
-         break;
-      case H_CURL_R1D:
-         switch (dim)
-         {
-            case 1:
-               // curl: 1D H_CURL_R1D -> H_DIV_R1D
-               deriv_type = CURL;
-               deriv_range_type = VECTOR;
-               deriv_map_type = H_DIV_R1D;
-               break;
-            case 0:
-               deriv_type = NONE;
-               deriv_range_type = UNKNOWN_RANGE_TYPE;
-               deriv_map_type = UNKNOWN_MAP_TYPE;
-            default:
-               MFEM_ABORT("Invalid dimension, Dim = " << dim);
-         }
-         break;
       default:
          MFEM_ABORT("Invalid MapType = " << map_type);
    }

fem/fe/fe_base.hpp

@@ -295,20 +295,10 @@ public:
                           $ u(x) = (1/w) \hat u(\hat x) $ */
       H_DIV,     /**< For vector fields; preserves surface integrals of the
                           normal component $ u(x) = (J/w) \hat u(\hat x) $ */
-      H_CURL,    /**< For vector fields; preserves line integrals of the
+      H_CURL     /**< For vector fields; preserves line integrals of the
                           tangential component
                           $ u(x) = J^{-t} \hat u(\hat x) $ (square J),
                           $ u(x) = J(J^t J)^{-1} \hat u(\hat x) $ (general J) */
-      H_DIV_R2D, /**< For 3-component vector fields in 2D; equivalent to a
-                          direct sum of an H_DIV basis and an INTEGRAL basis */
-      H_CURL_R2D,/**< For 3-component vector fields in 2D; equivalent to a
-                          direct sum of an H_CURL basis and a VALUE basis */
-      H_DIV_R1D, /**< For 3-component vector fields in 1D; equivalent to a
-                          direct sum of a VALUE basis and a pair of INTEGRAL
-                          bases */
-      H_CURL_R1D /**< For 3-component vector fields in 1D; equivalent to a
-                          direct sum of an INTEGRAL basis and a pair of VALUE
-                          bases */
    };
 
    /** @brief Enumeration for DerivType: defines which derivative method
@@ -340,28 +330,12 @@ public:
    int GetDim() const { return dim; }
 
    /** @brief Returns the vector dimension for vector-valued finite elements,
-       which is also the dimension of the interpolation operation and the
-       width of the DenseMatrix argument in
-       CalcVShape(const IntegrationPoint &ip, DenseMatrix &shape). */
+       which is also the dimension of the interpolation operation. */
    int GetRangeDim() const { return vdim; }
 
-   /** @brief Returns the vector dimension, in physical space, for
-       vector-valued finite elements, which is also the width of the
-       DenseMatrix argument in
-       CalcPhysVShape(ElementTransformation &Trans, DenseMatrix &shape). */
-   virtual int GetPhysRangeDim(int /* space_dim */) const { return vdim; }
-
-   /** Returns the dimension of the curl for vector-valued finite elements,
-       which is also the width of the DenseMatrix argument in
-       CalcCurlShape(const IntegrationPoint &ip, DenseMatrix &curl_shape). */
+   /// Returns the dimension of the curl for vector-valued finite elements.
    int GetCurlDim() const { return cdim; }
 
-   /** Returns the dimension, in physical space, of the curl for vector-valued
-       finite elements, which is also the width of the DenseMatrix argument in
-       CalcPhysCurlShape(ElementTransformation &Trans, DenseMatrix &curl_shape).
-   */
-   virtual int GetPhysCurlDim(int /* space_dim */) const { return cdim; }
-
    /// Returns the Geometry::Type of the reference element.
    Geometry::Type GetGeomType() const { return geom_type; }
 
@@ -1016,8 +990,6 @@ protected:
 public:
    VectorFiniteElement(int D, Geometry::Type G, int Do, int O, int M,
                        int F = FunctionSpace::Pk);
-
-   int GetPhysRangeDim(int space_dim) const override { return space_dim; }
 };
 
 /// @brief Class for computing 1D special polynomials and their associated basis

fem/fe/fe_nd.cpp

@@ -2531,7 +2531,7 @@ void ND_FuentesPyramidElement::calcCurlBasis(const int p,
 
 ND_R1D_PointElement::ND_R1D_PointElement(int p)
    : VectorFiniteElement(1, Geometry::POINT, 2, p,
-                         H_CURL_R1D, FunctionSpace::Pk)
+                         H_CURL, FunctionSpace::Pk)
 {
    // VectorFiniteElement::SetDerivMembers doesn't support 0D H_CURL elements
    // so we mimic a 1D element and then correct the dimension here.
@@ -2562,7 +2562,7 @@ ND_R1D_SegmentElement::ND_R1D_SegmentElement(const int p,
                                              const int cb_type,
                                              const int ob_type)
    : VectorFiniteElement(1, Geometry::SEGMENT, 3 * p + 2, p,
-                         H_CURL_R1D, FunctionSpace::Pk),
+                         H_CURL, FunctionSpace::Pk),
      dof2tk(dof),
      cbasis1d(poly1d.GetBasis(p, VerifyClosed(cb_type))),
      obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(ob_type)))
@@ -2839,7 +2839,7 @@ ND_R2D_SegmentElement::ND_R2D_SegmentElement(const int p,
                                              const int cb_type,
                                              const int ob_type)
    : VectorFiniteElement(1, Geometry::SEGMENT, 2 * p + 1, p,
-                         H_CURL_R2D, FunctionSpace::Pk),
+                         H_CURL, FunctionSpace::Pk),
      dof2tk(dof),
      cbasis1d(poly1d.GetBasis(p, VerifyClosed(cb_type))),
      obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(ob_type)))
@@ -3023,7 +3023,7 @@ void ND_R2D_SegmentElement::Project(VectorCoefficient &vc,
 ND_R2D_FiniteElement::ND_R2D_FiniteElement(int p, Geometry::Type G, int Do,
                                            const real_t *tk_fe)
    : VectorFiniteElement(2, G, Do, p,
-                         H_CURL_R2D, FunctionSpace::Pk),
+                         H_CURL, FunctionSpace::Pk),
      tk(tk_fe),
      dof_map(dof),
      dof2tk(dof)

fem/fe/fe_nd.hpp

@@ -663,9 +663,6 @@ public:
                          const int cb_type = BasisType::GaussLobatto,
                          const int ob_type = BasisType::GaussLegendre);
 
-   int GetPhysRangeDim(int space_dim) const override { return 2; }
-   int GetPhysCurlDim(int space_dim) const override { return 1; }
-
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
 
@@ -708,9 +705,6 @@ private:
                            DenseMatrix &I) const;
 
 public:
-   int GetPhysRangeDim(int space_dim) const override { return 3; }
-   int GetPhysCurlDim(int space_dim) const override { return 3; }
-
    using FiniteElement::CalcVShape;
    using FiniteElement::CalcPhysCurlShape;
 

fem/fe/fe_rt.cpp

@@ -2006,7 +2006,7 @@ RT_R1D_SegmentElement::RT_R1D_SegmentElement(const int p,
                                              const int cb_type,
                                              const int ob_type)
    : VectorFiniteElement(1, Geometry::SEGMENT, 3 * p + 4, p + 1,
-                         H_DIV_R1D, FunctionSpace::Pk),
+                         H_DIV, FunctionSpace::Pk),
      dof2nk(dof),
      cbasis1d(poly1d.GetBasis(p + 1, VerifyClosed(cb_type))),
      obasis1d(poly1d.GetBasis(p, VerifyOpen(ob_type)))
@@ -2281,7 +2281,7 @@ const real_t RT_R2D_SegmentElement::nk[2] = { 0.,1.};
 RT_R2D_SegmentElement::RT_R2D_SegmentElement(const int p,
                                              const int ob_type)
    : VectorFiniteElement(1, Geometry::SEGMENT, p + 1, p + 1,
-                         H_DIV_R2D, FunctionSpace::Pk),
+                         H_DIV, FunctionSpace::Pk),
      dof2nk(dof),
      obasis1d(poly1d.GetBasis(p, VerifyOpen(ob_type)))
 {
@@ -2392,7 +2392,7 @@ void RT_R2D_SegmentElement::LocalInterpolation(const VectorFiniteElement &cfe,
 RT_R2D_FiniteElement::RT_R2D_FiniteElement(int p, Geometry::Type G, int Do,
                                            const real_t *nk_fe)
    : VectorFiniteElement(2, G, Do, p + 1,
-                         H_DIV_R2D, FunctionSpace::Pk),
+                         H_DIV, FunctionSpace::Pk),
      nk(nk_fe),
      dof_map(dof),
      dof2nk(dof)

fem/fe/fe_rt.hpp

@@ -510,9 +510,6 @@ public:
    RT_R2D_SegmentElement(const int p,
                          const int ob_type = BasisType::GaussLegendre);
 
-   int GetPhysRangeDim(int space_dim) const override { return 2; }
-   int GetPhysCurlDim(int space_dim) const override { return 0; }
-
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
 
@@ -550,9 +547,6 @@ private:
                            DenseMatrix &I) const;
 
 public:
-   int GetPhysRangeDim(int space_dim) const override { return 3; }
-   int GetPhysCurlDim(int space_dim) const override { return 0; }
-
    using FiniteElement::CalcVShape;
 
    void CalcVShape(ElementTransformation &Trans,

fem/fespace.cpp

@@ -3934,16 +3934,6 @@ const FiniteElement *FiniteElementSpace::GetBE(int i) const
    return BE;
 }
 
-const FiniteElement *FiniteElementSpace::GetTypicalBE() const
-{
-   if (mesh->GetNBE() > 0) { return GetBE(0); }
-
-   Geometry::Type geom = mesh->GetTypicalFaceGeometry();
-   const FiniteElement *be = fec->FiniteElementForGeometry(geom);
-   MFEM_VERIFY(be != nullptr, "Could not determine a typical BE!");
-   return be;
-}
-
 const FiniteElement *FiniteElementSpace::GetFaceElement(int i) const
 {
    MFEM_VERIFY(!IsVariableOrder(), "not implemented");
@@ -3974,11 +3964,6 @@ const FiniteElement *FiniteElementSpace::GetFaceElement(int i) const
    return fe;
 }
 
-const FiniteElement *FiniteElementSpace::GetTypicalFaceElement() const
-{
-   return fec->FiniteElementForGeometry(mesh->GetTypicalFaceGeometry());
-}
-
 const FiniteElement *FiniteElementSpace::GetEdgeElement(int i,
                                                         int variant) const
 {

fem/fespace.hpp

@@ -839,7 +839,7 @@ public:
        Note: For vector-valued elements, the results pads up the range dimension
        to the spatial dimension. E.g., consider a stack of 5 vector-valued
        elements each representing 2D vectors, living in a 3 dimensional space.
-       Then this function would give 15, not 10.
+       Then this fucntion would give 15, not 10.
        */
    int GetVectorDim() const;
 
@@ -1323,24 +1323,12 @@ public:
         associated with i'th boundary face in the mesh object. */
    const FiniteElement *GetBE(int i) const;
 
-   /// @brief Return a "typical" boundary element.
-   ///
-   /// This can be used in situations where the local mesh partition may be
-   /// empty.
-   const FiniteElement *GetTypicalBE() const;
-
    /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
         associated with i'th face in the mesh object.  Faces in this case refer
         to the MESHDIM-1 primitive so in 2D they are segments and in 1D they are
         points.*/
    const FiniteElement *GetFaceElement(int i) const;
 
-   /// @brief Return a "typical" face element.
-   ///
-   /// This can be used in situations where the local mesh partition may be
-   /// empty.
-   const FiniteElement *GetTypicalFaceElement() const;
-
    /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
         associated with i'th edge in the mesh object. */
    const FiniteElement *GetEdgeElement(int i, int variant = 0) const;

fem/gridfunc.cpp

@@ -345,6 +345,27 @@ void GridFunction::ComputeFlux(BilinearFormIntegrator &blfi,
    }
 }
 
+int GridFunction::VectorDim() const
+{
+   const FiniteElement *fe = fes->GetTypicalFE();
+   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
+   {
+      return fes->GetVDim();
+   }
+   return fes->GetVDim()*std::max(fes->GetMesh()->SpaceDimension(),
+                                  fe->GetRangeDim());
+}
+
+int GridFunction::CurlDim() const
+{
+   const FiniteElement *fe = fes->GetTypicalFE();
+   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
+   {
+      return 2 * fes->GetMesh()->SpaceDimension() - 3;
+   }
+   return fes->GetVDim()*fe->GetCurlDim();
+}
+
 void GridFunction::GetTrueDofs(Vector &tv) const
 {
    const SparseMatrix *R = fes->GetRestrictionMatrix();
@@ -2029,18 +2050,6 @@ void GridFunction::AccumulateAndCountBdrValues(
    Coefficient *coeff[], VectorCoefficient *vcoeff, const Array<int> &attr,
    Array<int> &values_counter)
 {
-   if (vcoeff)
-   {
-      MFEM_VERIFY(fes->GetVDim() == vcoeff->GetVDim(),
-                  "vcoeff vdim != fes VDim");
-      MFEM_VERIFY(fes->GetTypicalBE()->GetMapType() == FiniteElement::VALUE &&
-                  fes->GetTypicalBE()->GetRangeType() ==
-                  FiniteElement::SCALAR,
-                  "Can only call ProjectBdrCoefficient on scalar value-type "
-                  "boundary elements. "
-                  "Did you intended to call ProjectBdrCoefficientNormal or "
-                  "ProjectBdrCoefficientTangent for vector finite elements?");
-   }
    Array<int> vdofs;
    Vector vc;
 
@@ -2193,9 +2202,6 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
    VectorCoefficient &vcoeff, const Array<int> &bdr_attr,
    Array<int> &values_counter)
 {
-   MFEM_VERIFY(fes->GetTypicalBE()->GetPhysRangeDim(
-                  fes->GetMesh()->SpaceDimension()) == vcoeff.GetVDim(),
-               "vcoeff vdim != PhysRangeDim");
    const FiniteElement *fe;
    ElementTransformation *T;
    Array<int> dofs;
@@ -2349,9 +2355,6 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
 
 void GridFunction::ProjectCoefficient(Coefficient &coeff, ProjectType type)
 {
-   MFEM_VERIFY(
-      VectorDim() == 1,
-      "Cannot project scalar Coefficient onto vector GridFunction");
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
    DofTransformation doftrans;
    Array<int> vdofs;
@@ -2627,7 +2630,6 @@ void GridFunction::ProjectCoefficient(
 void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff,
                                       ProjectType type)
 {
-   MFEM_VERIFY(VectorDim() == vcoeff.GetVDim(), "vcoeff vdim != VectorDim()");
    Array<int> vdofs;
    Vector vals;
    DofTransformation doftrans;
@@ -2943,7 +2945,6 @@ void GridFunction::ProjectCoefficientElementL2(VectorCoefficient &vcoeff)
 void GridFunction::ProjectCoefficient(
    VectorCoefficient &vcoeff, Array<int> &dofs)
 {
-   MFEM_VERIFY(VectorDim() == vcoeff.GetVDim(), "vcoeff vdim != VectorDim()");
    int el = -1;
    ElementTransformation *T = NULL;
    const FiniteElement *fe = NULL;
@@ -2973,7 +2974,6 @@ void GridFunction::ProjectCoefficient(
 
 void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff, int attribute)
 {
-   MFEM_VERIFY(VectorDim() == vcoeff.GetVDim(), "vcoeff vdim != VectorDim()");
    int i;
    Array<int> vdofs;
    Vector vals;
@@ -3030,14 +3030,9 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
    }
 }
 
-void GridFunction::ProjectDiscCoefficient(
-   std::variant<Coefficient*, VectorCoefficient*> coeff, Array<int> &dof_attr)
+void GridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff,
+                                          Array<int> &dof_attr)
 {
-   std::visit([&](auto* c)
-   {
-      MFEM_VERIFY(VectorDim() == c->GetVDim(), "coeff vdim != VectorDim()");
-   }, coeff);
-
    Array<int> vdofs;
    Vector vals;
 
@@ -3051,10 +3046,7 @@ void GridFunction::ProjectDiscCoefficient(
    {
       fes->GetElementVDofs(i, vdofs);
       vals.SetSize(vdofs.Size());
-      std::visit([&](auto* c)
-      {
-         fes->GetFE(i)->Project(*c, *fes->GetElementTransformation(i), vals);
-      }, coeff);
+      fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
 
       // the values in shared dofs are determined from the element with maximal
       // attribute
@@ -3070,15 +3062,17 @@ void GridFunction::ProjectDiscCoefficient(
    }
 }
 
+void GridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff)
+{
+   Array<int> dof_attr;
+   ProjectDiscCoefficient(coeff, dof_attr);
+}
+
 void GridFunction::ProjectDiscCoefficient(Coefficient &coeff, AvgType type)
 {
    // Harmonic  (x1 ... xn) = [ (1/x1 + ... + 1/xn) / n ]^-1.
    // Arithmetic(x1 ... xn) = (x1 + ... + xn) / n.
 
-   MFEM_VERIFY(
-      VectorDim() == 1,
-      "Cannot project a scalar coefficient onto a vector GridFunction");
-
    Array<int> zones_per_vdof;
    AccumulateAndCountZones(coeff, type, zones_per_vdof);
 
@@ -3088,7 +3082,6 @@ void GridFunction::ProjectDiscCoefficient(Coefficient &coeff, AvgType type)
 void GridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff,
                                           AvgType type)
 {
-   MFEM_VERIFY(VectorDim() == coeff.GetVDim(), "coeff vdim != VectorDim()");
    Array<int> zones_per_vdof;
    AccumulateAndCountZones(coeff, type, zones_per_vdof);
 
@@ -3144,33 +3137,52 @@ void GridFunction::ProjectBdrCoefficient(Coefficient *coeff[],
 }
 
 void GridFunction::ProjectBdrCoefficientNormal(
-   Coefficient *coeff, VectorCoefficient *vcoeff, const Array<int> &bdr_attr)
+   VectorCoefficient &vcoeff, const Array<int> &bdr_attr)
 {
-   MFEM_VERIFY(fes->GetVDim() == 1, "fespace VDim != 1");
-   MFEM_VERIFY(fes->GetTypicalBE()->GetRangeType() == FiniteElement::SCALAR &&
-               fes->GetTypicalBE()->GetMapType() == FiniteElement::INTEGRAL,
-               "Not an RT FE space!");
-   if (vcoeff)
-   {
-      MFEM_VERIFY(vcoeff->GetVDim() == fes->GetMesh()->SpaceDimension(),
-                  "vcoeff vdim (" << vcoeff->GetVDim()
-                  << ") != SpaceDimension ("
-                  << fes->GetMesh()->SpaceDimension() << ")");
-   }
+#if 0
+   // implementation for the case when the face dofs are integrals of the
+   // normal component.
+   const FiniteElement *fe;
+   ElementTransformation *T;
+   Array<int> dofs;
+   int dim = vcoeff.GetVDim();
+   Vector vc(dim), nor(dim), lvec, shape;
 
+   for (int i = 0; i < fes->GetNBE(); i++)
+   {
+      if (bdr_attr[fes->GetBdrAttribute(i)-1] == 0)
+      {
+         continue;
+      }
+      fe = fes->GetBE(i);
+      T = fes->GetBdrElementTransformation(i);
+      int intorder = 2*fe->GetOrder(); // !!!
+      const IntegrationRule &ir = IntRules.Get(fe->GetGeomType(), intorder);
+      int nd = fe->GetDof();
+      lvec.SetSize(nd);
+      shape.SetSize(nd);
+      lvec = 0.0;
+      for (int j = 0; j < ir.GetNPoints(); j++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(j);
+         T->SetIntPoint(&ip);
+         vcoeff.Eval(vc, *T, ip);
+         CalcOrtho(T->Jacobian(), nor);
+         fe->CalcShape(ip, shape);
+         lvec.Add(ip.weight * (vc * nor), shape);
+      }
+      fes->GetBdrElementDofs(i, dofs);
+      SetSubVector(dofs, lvec);
+   }
+#else
    // implementation for the case when the face dofs are scaled point
    // values of the normal component.
    const FiniteElement *fe;
    ElementTransformation *T;
    Array<int> dofs;
-   Vector vc, nor, lvec;
+   int dim = vcoeff.GetVDim();
+   Vector vc(dim), nor(dim), lvec;
    DofTransformation doftrans;
-   if (vcoeff)
-   {
-      const int dim = vcoeff->GetVDim();
-      vc.SetSize(dim);
-      nor.SetSize(dim);
-   }
 
    for (int i = 0; i < fes->GetNBE(); i++)
    {
@@ -3186,22 +3198,15 @@ void GridFunction::ProjectBdrCoefficientNormal(
       {
          const IntegrationPoint &ip = ir.IntPoint(j);
          T->SetIntPoint(&ip);
-         if (coeff)
-         {
-            const real_t c = coeff->Eval(*T, ip);
-            lvec(j) = c * T->Weight();
-         }
-         else if (vcoeff)
-         {
-            vcoeff->Eval(vc, *T, ip);
-            CalcOrtho(T->Jacobian(), nor);
-            lvec(j) = (vc * nor);
-         }
+         vcoeff.Eval(vc, *T, ip);
+         CalcOrtho(T->Jacobian(), nor);
+         lvec(j) = (vc * nor);
       }
       fes->GetBdrElementDofs(i, dofs, doftrans);
       doftrans.TransformPrimal(lvec);
       SetSubVector(dofs, lvec);
    }
+#endif
 }
 
 void GridFunction::ProjectBdrCoefficientTangent(
@@ -5002,14 +5007,6 @@ real_t ExtrudeCoefficient::Eval(ElementTransformation &T,
    return sol_in.Eval(*T_in, ip);
 }
 
-void VectorExtrudeCoefficient::Eval(Vector &v, ElementTransformation &T,
-                                    const IntegrationPoint &ip)
-{
-   ElementTransformation *T_in =
-      mesh_in->GetElementTransformation(T.ElementNo / n);
-   T_in->SetIntPoint(&ip);
-   sol_in.Eval(v, *T_in, ip);
-}
 
 GridFunction *Extrude1DGridFunction(Mesh *mesh, Mesh *mesh2d,
                                     GridFunction *sol, const int ny)
@@ -5060,17 +5057,10 @@ GridFunction *Extrude1DGridFunction(Mesh *mesh, Mesh *mesh2d,
       return NULL;
    }
    FiniteElementSpace *solfes2d;
-   const int vdim = sol->FESpace()->GetVDim();
-   solfes2d = new FiniteElementSpace(mesh2d, solfec2d, vdim);
+   // assuming sol is scalar
+   solfes2d = new FiniteElementSpace(mesh2d, solfec2d);
    sol2d = new GridFunction(solfes2d);
    sol2d->MakeOwner(solfec2d);
-   if (vdim > 1)
-   {
-      VectorGridFunctionCoefficient vcsol(sol);
-      VectorExtrudeCoefficient vc2d(mesh, vcsol, ny);
-      sol2d->ProjectCoefficient(vc2d);
-   }
-   else
    {
       GridFunctionCoefficient csol(sol);
       ExtrudeCoefficient c2d(mesh, csol, ny);
@@ -5252,30 +5242,6 @@ void GridFunction::GetElementBounds(const PLBound &plb,
                                     Vector &lower, Vector &upper,
                                     const int vdim) const
 {
-   if (UseDevice() && Device::Allows(Backend::DEVICE_MASK) &&
-       plb.GetBasisType() != BasisType::Positive &&
-       UsesTensorBasis(*fes))
-   {
-      const FiniteElement &fe = *fes->GetTypicalFE();
-      const int rdim = fe.GetDim();
-      const int fes_dim = fes->GetVDim();
-      const int nel = fes->GetNE();
-      const int nd = fe.GetDof();
-
-      Vector e_vec(nd*fes_dim*nel, Device::GetDeviceMemoryType());
-      e_vec.UseDevice(true);
-      const ElementRestrictionOperator *elem_restr =
-         fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC);
-      MFEM_VERIFY(elem_restr != nullptr,
-                  "Element restriction is required for device bounds.");
-      elem_restr->Mult(*this, e_vec);
-
-      plb.GetElementBoundsKernel(rdim, fes_dim, e_vec, lower, upper, vdim);
-      lower.HostRead();
-      upper.HostRead();
-      return;
-   }
-
    int nel  = fes->GetNE();
    int fes_dim  = fes->GetVDim();
    lower.SetSize(nel*(vdim > 0 ? 1 :fes_dim));
@@ -5792,4 +5758,4 @@ std::pair<real_t, real_t> GridFunction::EstimateFunctionMaximum(
    return std::make_pair(global_max_lower, global_max_upper);
 }
 
-}
+}

fem/gridfunc.hpp

@@ -23,7 +23,6 @@
 #include <limits>
 #include <ostream>
 #include <string>
-#include <variant>
 
 namespace mfem
 {
@@ -80,18 +79,10 @@ protected:
                         bool wcoef,
                         int subdomain);
 
-   /** @brief Project a discontinuous (vector) coefficient as a grid function on
-       a continuous finite element space. Return in dof_attr the maximal
-       attribute of the elements containing each degree of freedom. */
-   virtual void ProjectDiscCoefficient(
-      std::variant<Coefficient*, VectorCoefficient*> coeff, Array<int> &dof_attr);
-
-   /** @brief Project a discontinuous (vector) coefficient as a grid function on
-       a continuous finite element space. The values in shared dofs are
-       determined from the element with maximal attribute. */
-   virtual void ProjectDiscCoefficient(
-      std::variant<Coefficient*, VectorCoefficient*> coeff)
-   { Array<int> dof_attr; ProjectDiscCoefficient(coeff, dof_attr); };
+   /** Project a discontinuous vector coefficient in a continuous space and
+       return in dof_attr the maximal attribute of the elements containing each
+       degree of freedom. */
+   void ProjectDiscCoefficient(VectorCoefficient &coeff, Array<int> &dof_attr);
 
    /** Helper function for ProjectCoefficientElementL2 */
    void ProjectCoefficientElementL2_(Coefficient &coeff, Vector &sol, Vector &Va);
@@ -159,13 +150,11 @@ public:
 
    FiniteElementCollection *OwnFEC() { return fec_owned; }
 
-   /** @brief Shortcut for calling FiniteElementSpace::GetVectorDim() on the
-       underlying #fes */
-   int VectorDim() const { return fes->GetVectorDim(); }
+   /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the underlying #fes
+   int VectorDim() const;
 
-   /** @brief Shortcut for calling FiniteElementSpace::GetCurlDim() on the
-       underlying #fes */
-   int CurlDim() const { return fes->GetCurlDim(); }
+   /// Shortcut for calling FiniteElementSpace::GetCurlDim() on the underlying #fes
+   int CurlDim() const;
 
    /// Read only access to the (optional) internal true-dof Vector.
    const Vector &GetTrueVector() const
@@ -524,17 +513,10 @@ public:
        but using an array of scalar coefficients for each component. */
    void ProjectCoefficient(Coefficient *coeff[]);
 
-   /** @brief Project a discontinuous coefficient as a grid function on
-       a continuous finite element space. The values in shared dofs are
-       determined from the element with maximal attribute. */
-   virtual void ProjectDiscCoefficient(Coefficient &coeff)
-   { ProjectDiscCoefficient(&coeff); }
-
    /** @brief Project a discontinuous vector coefficient as a grid function on
        a continuous finite element space. The values in shared dofs are
        determined from the element with maximal attribute. */
-   virtual void ProjectDiscCoefficient(VectorCoefficient &coeff)
-   { ProjectDiscCoefficient(&coeff); }
+   virtual void ProjectDiscCoefficient(VectorCoefficient &coeff);
 
    enum AvgType {ARITHMETIC, HARMONIC};
    /** @brief Projects a discontinuous coefficient so that the values in shared
@@ -550,9 +532,6 @@ public:
    std::unique_ptr<GridFunction> ProlongateToMaxOrder() const;
 
 protected:
-   void ProjectBdrCoefficientNormal(Coefficient *coeff, VectorCoefficient *vcoeff,
-                                    const Array<int> &attr);
-
    /** @brief Accumulates (depending on @a type) the values of @a coeff at all
        shared vdofs and counts in how many zones each vdof appears. */
    void AccumulateAndCountZones(Coefficient &coeff, AvgType type,
@@ -677,26 +656,15 @@ public:
    virtual void ProjectBdrCoefficient(Coefficient *coeff[],
                                       const Array<int> &attr);
 
-   /** @brief Project the normal component of the given VectorCoefficient on
-       the boundary. */
-   /** Only boundary attributes that are marked in @a bdr_attr are
-       projected. Assumes RT-type vector finite element GridFunction. */
+   /** Project the normal component of the given VectorCoefficient on
+       the boundary. Only boundary attributes that are marked in
+       'bdr_attr' are projected. Assumes RT-type VectorFE GridFunction. */
    void ProjectBdrCoefficientNormal(VectorCoefficient &vcoeff,
-                                    const Array<int> &bdr_attr)
-   { ProjectBdrCoefficientNormal(NULL, &vcoeff, bdr_attr); }
-
-   /** @brief Project the given Coefficient in the normal direction on the
-       boundary. */
-   /** Only boundary attributes that are marked in @a bdr_attr are projected.
-       Assumes RT-type vector finite element GridFunction. */
-   void ProjectBdrCoefficientNormal(Coefficient &coeff,
-                                    const Array<int> &bdr_attr)
-   { ProjectBdrCoefficientNormal(&coeff, NULL, bdr_attr); }
+                                    const Array<int> &bdr_attr);
 
    /** @brief Project the tangential components of the given VectorCoefficient
-       on the boundary. */
-   /** Only boundary attributes that are marked in @a bdr_attr
-       are projected. Assumes ND-type vector finite element GridFunction. */
+       on the boundary. Only boundary attributes that are marked in @a bdr_attr
+       are projected. Assumes ND-type VectorFE GridFunction. */
    virtual void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                              const Array<int> &bdr_attr);
 
@@ -1946,7 +1914,7 @@ real_t ComputeElementLpDistance(real_t p, int i,
                                 GridFunction& gf1, GridFunction& gf2);
 
 
-/// Class used for extruding a scalar coefficient
+/// Class used for extruding scalar GridFunctions
 class ExtrudeCoefficient : public Coefficient
 {
 private:
@@ -1954,53 +1922,13 @@ private:
    Mesh *mesh_in;
    Coefficient &sol_in;
 public:
-   /// Constructs an instance of VectorExtrudeCoefficient
-   /**
-    * @param m      1D mesh
-    * @param s      1D vector coefficient
-    * @param n_     number of transverse elements of the extruded mesh
-    */
    ExtrudeCoefficient(Mesh *m, Coefficient &s, int n_)
-      : n(n_), mesh_in(m), sol_in(s)
-   { MFEM_VERIFY(n > 0, "Number of transverse elements must be positive!"); }
-
+      : n(n_), mesh_in(m), sol_in(s) { }
    real_t Eval(ElementTransformation &T, const IntegrationPoint &ip) override;
-
    virtual ~ExtrudeCoefficient() { }
 };
 
-/// Class used for extruding a vector coefficient
-class VectorExtrudeCoefficient : public VectorCoefficient
-{
-private:
-   int n;
-   Mesh *mesh_in;
-   VectorCoefficient &sol_in;
-public:
-   /// Constructs an instance of VectorExtrudeCoefficient
-   /**
-    * @param m      1D mesh
-    * @param s      1D vector coefficient
-    * @param n_     number of transverse elements of the extruded mesh
-    */
-   VectorExtrudeCoefficient(Mesh *m, VectorCoefficient &s, int n_)
-      : VectorCoefficient(s.GetVDim()), n(n_), mesh_in(m), sol_in(s)
-   { MFEM_VERIFY(n > 0, "Number of transverse elements must be positive!"); }
-
-   void Eval(Vector &v, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
-   using VectorCoefficient::Eval;
-
-   virtual ~VectorExtrudeCoefficient() { }
-};
-
-/// Extrude a 1D GridFunction, after extruding the mesh with Extrude1D()
-/**
- * @param mesh      1D mesh
- * @param mesh2d    extruded mesh
- * @param sol       grid function
- * @param ny        number of transverse elements of the extruded mesh
- */
+/// Extrude a scalar 1D GridFunction, after extruding the mesh with Extrude1D.
 GridFunction *Extrude1DGridFunction(Mesh *mesh, Mesh *mesh2d,
                                     GridFunction *sol, const int ny);
 

fem/gslib.cpp

@@ -106,7 +106,9 @@ FindPointsGSLIB::FindPointsGSLIB()
    gsl_comm = new gslib::comm;
    cr       = new gslib::crystal;
 #ifdef MFEM_USE_MPI
-   if (!Mpi::IsInitialized()) { Mpi::Init(); }
+   int initialized = 0;
+   MPI_Initialized(&initialized);
+   if (!initialized) { MPI_Init(NULL, NULL); }
    MPI_Comm comm = MPI_COMM_WORLD;
    comm_init(gsl_comm, comm);
 #else
@@ -488,7 +490,7 @@ void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
    }
    DEV.find_device = true;
 
-   const unsigned int id = gsl_comm->id, np = gsl_comm->np;
+   const int id = gsl_comm->id, np = gsl_comm->np;
 
    gsl_mfem_ref.SetSize(points_cnt * dim);
    gsl_mfem_elem.SetSize(points_cnt);
@@ -650,7 +652,7 @@ void FindPointsGSLIB::FindPointsOnDevice(const Vector &point_pos,
          {
             const int pp = hash_offset[i];
             /* don't send back to where it just came from */
-            if (static_cast<unsigned>(pp) == p->proc)
+            if (pp == p->proc)
             {
                continue;
             }
@@ -1066,7 +1068,7 @@ void FindPointsGSLIB::InterpolateOnDevice(const Vector &field_in_evec,
          sarray_transfer(struct evalOutPt_t, &outpt, proc, 1, cr);
 
          opt = (evalOutPt_t *)outpt.ptr;
-         for (size_t index = 0; index < outpt.n; index++)
+         for (int index = 0; index < outpt.n; index++)
          {
             int idx = ordering == Ordering::byNODES ?
                       opt->index + i*points_cnt :
@@ -1411,7 +1413,7 @@ void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order)
 {
    MFEM_VERIFY(mesh, "Setup FindPointsGSLIB with mesh first.");
    const int dof1D = order+1;
-   dim = mesh->Dimension();
+   const int dim = mesh->Dimension();
 
    SetupSplitMeshes();
    if (dim == 2)
@@ -2252,8 +2254,7 @@ void FindPointsGSLIB::DistributeInterpolatedValues(const Vector &int_vals,
       sarray_transfer(struct out_pt, outpt, proc, 1, cr);
 
       // Store received data
-      MFEM_VERIFY(outpt->n == static_cast<size_t>(points_cnt),
-                  "Incompatible size. Number of points "
+      MFEM_VERIFY(outpt->n == points_cnt, "Incompatible size. Number of points "
                   "received does not match the number of points originally "
                   "found using FindPoints.");
 
@@ -2622,7 +2623,9 @@ GSOPGSLIB::GSOPGSLIB(Array<long long> &ids)
    gsl_comm = new gslib::comm;
    cr       = new gslib::crystal;
 #ifdef MFEM_USE_MPI
-   if (!Mpi::IsInitialized()) { Mpi::Init(); }
+   int initialized;
+   MPI_Initialized(&initialized);
+   if (!initialized) { MPI_Init(NULL, NULL); }
    MPI_Comm comm = MPI_COMM_WORLD;
    comm_init(gsl_comm, comm);
 #else

fem/gslib.hpp

@@ -202,19 +202,13 @@ protected:
                             const int dof1dsol, const int ordering);
 
 public:
-   /// Serial constructor
    FindPointsGSLIB();
-
-   /// Serial constructor + setup with given Mesh (see \ref Setup)
    FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
 #ifdef MFEM_USE_MPI
-   /// Constructor for ParMesh
    FindPointsGSLIB(MPI_Comm comm_);
-
-   /// Constructor + setup with given ParMesh (see \ref Setup)
    FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);

fem/gslib/findpts_local_2.cpp

@@ -254,7 +254,7 @@ get_edge(const double *elx[2], const double *wtend, int ei,
       edge.dxdn[d] = workspace + (2 + d) * pN; //dxdn and dydn at DOFs along edge
    }
 
-   if (static_cast<unsigned>(side_init) != (1u << ei))
+   if (side_init != (1u << ei))
    {
 #define ELX(d, j, k) elx[d][j + k * pN] // assumes lexicographic ordering
       for (int d = 0; d < 2; ++d)

fem/gslib/findpts_local_3.cpp

@@ -294,7 +294,7 @@ get_face(const double *elx[3], const double *wtend, int fi, double *workspace,
       face.dxdn[d] = workspace+(3+d)*p_Nfr;
    }
 
-   if (static_cast<unsigned>(side_init) != (1u << fi))
+   if (side_init != (1u << fi))
    {
       const int e_stride[3] = {1, pN, pN*pN};
 #define ELX(d, j, k, l) elx[d][j*e_stride[d1]+k*e_stride[d2]+l*e_stride[dn]]
@@ -342,7 +342,7 @@ get_edge(const double *elx[3], const double *wtend, int ei, double *workspace,
 
    if (jidx >= 3*pN) { return edge; }
 
-   if (static_cast<unsigned>(side_init) != (64u << ei))
+   if (side_init != (64u << ei))
    {
       const int e_stride[3] = {1, pN, pN*pN};
 #define ELX(d, j, k, l) elx[d][j*e_stride[de]+k*e_stride[dn1]+l*e_stride[dn2]]

fem/integ/bilininteg_hdiv_ea.cpp

@@ -197,21 +197,15 @@ static void EAHdivAssemble3D(const int NE,
       // Assemble (one row per thread)
       MFEM_FOREACH_THREAD(idx_i, x, NDOF)
       {
-         // NOTE: due to an llvm backend bug, usage of the modulus operator
-         // has been removed from this foreach section.
          const int ic = idx_i / NDOF_C;
-         const int idx_ii = idx_i - ic * NDOF_C; // idx_i % NDOF_C
+         const int idx_ii = idx_i % NDOF_C;
 
          const int nx_i = (ic == 0) ? D1D : D1D-1;
          const int ny_i = (ic == 1) ? D1D : D1D-1;
 
-         const int qx_i = idx_ii / nx_i;
-         const int ix = idx_ii - qx_i * nx_i; // idx_ii % nx_i
-
-         const int qy_i = qx_i / ny_i;
-         const int iy = qx_i - qy_i * ny_i; // (idx_ii / nx_i) % ny_i
-
-         const int iz = qy_i; // (idx_ii / nx_i) / ny_i
+         const int ix = idx_ii % nx_i;
+         const int iy = (idx_ii / nx_i) % ny_i;
+         const int iz = (idx_ii / nx_i) / ny_i;
 
          const real_t (&Bi1)[MQ1][MD1] = (ic == 0) ? r_Bc : r_Bo;
          const real_t (&Bi2)[MQ1][MD1] = (ic == 1) ? r_Bc : r_Bo;
@@ -220,18 +214,14 @@ static void EAHdivAssemble3D(const int NE,
          for (int idx_j = 0; idx_j < NDOF; ++idx_j)
          {
             const int jc = idx_j / NDOF_C;
-            const int idx_jj = idx_j - jc * NDOF_C; // idx_j % NDOF_C
+            const int idx_jj = idx_j % NDOF_C;
 
             const int nx_j = (jc == 0) ? D1D : D1D-1;
             const int ny_j = (jc == 1) ? D1D : D1D-1;
 
-            const int qx_j = idx_jj / nx_j;
-            const int jx = idx_jj - qx_j * nx_j; // idx_jj % nx_j
-
-            const int qy_j = qx_j / ny_j;
-            const int jy = qx_j - qy_j * ny_j; // (idx_jj / nx_j) % ny_j
-
-            const int jz = qy_j; // (idx_jj / nx_j) / ny_j
+            const int jx = idx_jj % nx_j;
+            const int jy = (idx_jj / nx_j) % ny_j;
+            const int jz = (idx_jj / nx_j) / ny_j;
 
             const real_t (&Bj1)[MQ1][MD1] = (jc == 0) ? r_Bc : r_Bo;
             const real_t (&Bj2)[MQ1][MD1] = (jc == 1) ? r_Bc : r_Bo;

fem/integ/bilininteg_mass_kernels.hpp

@@ -181,12 +181,6 @@ constexpr int NBZ(int D1D)
 {
    return ipow(2, D(D1D) >= 0 ? D(D1D) : 0);
 }
-constexpr int NBZ3D(int MDQ)
-{
-   return MDQ > 0 ? std::min<int>(
-             (128 + MDQ * MDQ * MDQ - 1) / (MDQ * MDQ * MDQ), 64)
-          : 1;
-}
 }
 
 // Shared memory PA Mass Diagonal 2D kernel
@@ -810,23 +804,19 @@ void PAMassApply3D_Element(const int e,
    }
 }
 
-template <int T_D1D, int T_Q1D, int TBATCH, bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline void
-SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
-                          const real_t *d_, const real_t *x_, real_t *y_,
-                          int d1d = 0, int q1d = 0)
+template<int T_D1D, int T_Q1D, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void SmemPAMassApply3D_Element(const int e,
+                               const int NE,
+                               const real_t *b_,
+                               const real_t *d_,
+                               const real_t *x_,
+                               real_t *y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
 {
-   static_assert(TBATCH > 0, "TBATCH must be positive");
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-   constexpr int tbatch = TBATCH;
-   const int tidz = MFEM_THREAD_ID(z);
-#else
-   // host always batch size 1
-   constexpr int tbatch = 1;
-   constexpr int tidz = 0;
-#endif
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int D1D = T_D1D ? T_D1D : d1d;
+   constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
    constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
@@ -839,37 +829,33 @@ SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
    MFEM_SHARED real_t sDQ[MQ1*MD1];
    real_t (*B)[MD1] = (real_t (*)[MD1]) sDQ;
    real_t (*Bt)[MQ1] = (real_t (*)[MQ1]) sDQ;
-   MFEM_SHARED real_t sm0[tbatch][MDQ*MDQ*MDQ];
-   MFEM_SHARED real_t sm1[tbatch][MDQ*MDQ*MDQ];
-   real_t (*X)[MD1][MD1]   = (real_t (*)[MD1][MD1]) (sm0+tidz);
-   real_t (*DDQ)[MD1][MQ1] = (real_t (*)[MD1][MQ1]) (sm1+tidz);
-   real_t (*DQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) (sm0+tidz);
-   real_t (*QQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) (sm1+tidz);
-   real_t (*QQD)[MQ1][MD1] = (real_t (*)[MQ1][MD1]) (sm0+tidz);
-   real_t (*QDD)[MD1][MD1] = (real_t (*)[MD1][MD1]) (sm1+tidz);
-   MFEM_FOREACH_THREAD(dy, y, D1D)
+   MFEM_SHARED real_t sm0[MDQ*MDQ*MDQ];
+   MFEM_SHARED real_t sm1[MDQ*MDQ*MDQ];
+   real_t (*X)[MD1][MD1]   = (real_t (*)[MD1][MD1]) sm0;
+   real_t (*DDQ)[MD1][MQ1] = (real_t (*)[MD1][MQ1]) sm1;
+   real_t (*DQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) sm0;
+   real_t (*QQQ)[MQ1][MQ1] = (real_t (*)[MQ1][MQ1]) sm1;
+   real_t (*QQD)[MQ1][MD1] = (real_t (*)[MQ1][MD1]) sm0;
+   real_t (*QDD)[MD1][MD1] = (real_t (*)[MD1][MD1]) sm1;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
    {
-      MFEM_FOREACH_THREAD(dx, x, D1D)
+      MFEM_FOREACH_THREAD(dx,x,D1D)
       {
          MFEM_UNROLL(MD1)
          for (int dz = 0; dz < D1D; ++dz)
          {
-            X[dz][dy][dx] = x(dx, dy, dz, e);
+            X[dz][dy][dx] = x(dx,dy,dz,e);
          }
       }
-      MFEM_FOREACH_THREAD(dx, x, Q1D) { B[dx][dy] = b(dx, dy); }
-   }
-   if (tidz == 0)
-   {
-      MFEM_FOREACH_THREAD(dy, y, D1D)
+      MFEM_FOREACH_THREAD(dx,x,Q1D)
       {
-         MFEM_FOREACH_THREAD(dx, x, Q1D) { B[dx][dy] = b(dx, dy); }
+         B[dx][dy] = b(dx,dy);
       }
    }
    MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy, y, D1D)
+   MFEM_FOREACH_THREAD(dy,y,D1D)
    {
-      MFEM_FOREACH_THREAD(qx, x, Q1D)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
          real_t u[D1D];
          MFEM_UNROLL(MD1)
@@ -894,9 +880,9 @@ SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
       }
    }
    MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy, y, Q1D)
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
    {
-      MFEM_FOREACH_THREAD(qx, x, Q1D)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
          real_t u[D1D];
          MFEM_UNROLL(MD1)
@@ -921,9 +907,9 @@ SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
       }
    }
    MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy, y, Q1D)
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
    {
-      MFEM_FOREACH_THREAD(qx, x, Q1D)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
          real_t u[Q1D];
          MFEM_UNROLL(MQ1)
@@ -943,22 +929,22 @@ SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
          MFEM_UNROLL(MQ1)
          for (int qz = 0; qz < Q1D; qz++)
          {
-            QQQ[qz][qy][qx] = u[qz] * d(qx, qy, qz, e);
+            QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
          }
       }
    }
    MFEM_SYNC_THREAD;
-   if (tidz == 0)
+   MFEM_FOREACH_THREAD(di,y,D1D)
    {
-      MFEM_FOREACH_THREAD(di, y, D1D)
+      MFEM_FOREACH_THREAD(q,x,Q1D)
       {
-         MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[di][q] = b(q, di); }
+         Bt[di][q] = b(q,di);
       }
    }
    MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy, y, Q1D)
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
    {
-      MFEM_FOREACH_THREAD(dx, x, D1D)
+      MFEM_FOREACH_THREAD(dx,x,D1D)
       {
          real_t u[Q1D];
          MFEM_UNROLL(MQ1)
@@ -983,9 +969,9 @@ SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
       }
    }
    MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy, y, D1D)
+   MFEM_FOREACH_THREAD(dy,y,D1D)
    {
-      MFEM_FOREACH_THREAD(dx, x, D1D)
+      MFEM_FOREACH_THREAD(dx,x,D1D)
       {
          real_t u[Q1D];
          MFEM_UNROLL(MQ1)
@@ -1010,9 +996,9 @@ SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
       }
    }
    MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy, y, D1D)
+   MFEM_FOREACH_THREAD(dy,y,D1D)
    {
-      MFEM_FOREACH_THREAD(dx, x, D1D)
+      MFEM_FOREACH_THREAD(dx,x,D1D)
       {
          real_t u[D1D];
          MFEM_UNROLL(MD1)
@@ -1034,11 +1020,11 @@ SmemPAMassApply3D_Element(const int e, const int NE, const real_t *b_,
          {
             if (ACCUMULATE)
             {
-               y(dx, dy, dz, e) += u[dz];
+               y(dx,dy,dz,e) += u[dz];
             }
             else
             {
-               y(dx, dy, dz, e) = u[dz];
+               y(dx,dy,dz,e) = u[dz];
             }
          }
       }
@@ -1129,8 +1115,8 @@ inline void PAMassApply3D(const int NE,
    });
 }
 
-// Shared memory PA Mass Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int TBATCH=1>
+// Shared memory PA Mass Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
 inline void SmemPAMassApply3D(const int NE,
                               const Array<real_t> &b_,
                               const Array<real_t> &bt_,
@@ -1140,9 +1126,6 @@ inline void SmemPAMassApply3D(const int NE,
                               const int d1d = 0,
                               const int q1d = 0)
 {
-   static_assert(T_D1D > 0, "T_D1D must be positive");
-   static_assert(T_Q1D > 0, "T_Q1D must be positive");
-   static_assert(TBATCH > 0, "TBATCH must be positive");
    MFEM_CONTRACT_VAR(bt_);
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1154,11 +1137,9 @@ inline void SmemPAMassApply3D(const int NE,
    const auto d = d_.Read();
    const auto x = x_.Read();
    auto y = y_.ReadWrite();
-   mfem::forall_2D_batch<T_Q1D * T_Q1D * TBATCH>(NE, Q1D, Q1D, TBATCH,
-                                                 [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      internal::SmemPAMassApply3D_Element<T_D1D, T_Q1D, TBATCH>(e, NE, b, d, x,
-                                                                y, d1d, q1d);
+      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
    });
 }
 
@@ -1413,16 +1394,7 @@ ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
 {
    if constexpr (DIM == 1) { return internal::PAMassApply1D; }
    else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3)
-   {
-      constexpr int MDQ = T_D1D >= T_Q1D ? T_D1D : T_Q1D;
-      // max 64 threads in z limit in cuda and hip
-      if constexpr (MDQ > 0)
-      {
-         return internal::SmemPAMassApply3D<T_D1D, T_Q1D,
-                internal::mass::NBZ3D(MDQ)>;
-      }
-   }
+   else if constexpr (DIM == 3) { return internal::SmemPAMassApply3D<T_D1D, T_Q1D>; }
    MFEM_ABORT("");
 }
 

fem/intrules.hpp

@@ -43,52 +43,56 @@ public:
       index = i;
    }
 
-   void Set3w(const real_t x1, const real_t x2, const real_t x3, const real_t w)
-   { x = x1; y = x2; z = x3; weight = w; }
-   void Set2w(const real_t x1, const real_t x2, const real_t w)
-   { x = x1; y = x2; weight = w; }
-   void Set1w(const real_t x1, const real_t w)
-   { x = x1; weight = w; }
-
-   void Set3w(const real_t *p) { Set3w(p[0], p[1], p[2], p[3]); }
-   void Set2w(const real_t *p) { Set2w(p[0], p[1], p[2]); }
-   void Set1w(const real_t *p) { Set1w(p[0], p[1]); }
-
-   void Set3(const real_t x1, const real_t x2, const real_t x3)
-   { x = x1; y = x2; z = x3; }
-   void Set2(const real_t x1, const real_t x2)
-   { x = x1; y = x2; }
-   void Set1(const real_t x1)
-   { x = x1; }
-
-   void Set3(const real_t *p) { Set3(p[0], p[1], p[2]); }
-   void Set2(const real_t *p) { Set2(p[0], p[1]); }
-   void Set1(const real_t *p) { Set1(p[0]); }
-
-   void Set(const real_t x1, const real_t x2, const real_t x3, const real_t w)
-   { Set3w(x1, x2, x3, w); }
-
    void Set(const real_t *p, const int dim)
    {
       MFEM_ASSERT(1 <= dim && dim <= 3, "invalid dim: " << dim);
-      switch (dim)
+      x = p[0];
+      if (dim > 1)
       {
-         case 3: Set3(p); break;
-         case 2: Set2(p); break;
-         case 1: Set1(p); break;
+         y = p[1];
+         if (dim > 2)
+         {
+            z = p[2];
+         }
       }
    }
 
    void Get(real_t *p, const int dim) const
    {
       MFEM_ASSERT(1 <= dim && dim <= 3, "invalid dim: " << dim);
-      switch (dim)
+      p[0] = x;
+      if (dim > 1)
       {
-         case 3: p[2] = z;
-         case 2: p[1] = y;
-         case 1: p[0] = x;
+         p[1] = y;
+         if (dim > 2)
+         {
+            p[2] = z;
+         }
       }
    }
+
+   void Set(const real_t x1, const real_t x2, const real_t x3, const real_t w)
+   { x = x1; y = x2; z = x3; weight = w; }
+
+   void Set3w(const real_t *p) { x = p[0]; y = p[1]; z = p[2]; weight = p[3]; }
+
+   void Set3(const real_t x1, const real_t x2, const real_t x3)
+   { x = x1; y = x2; z = x3; }
+
+   void Set3(const real_t *p) { x = p[0]; y = p[1]; z = p[2]; }
+
+   void Set2w(const real_t x1, const real_t x2, const real_t w)
+   { x = x1; y = x2; weight = w; }
+
+   void Set2w(const real_t *p) { x = p[0]; y = p[1]; weight = p[2]; }
+
+   void Set2(const real_t x1, const real_t x2) { x = x1; y = x2; }
+
+   void Set2(const real_t *p) { x = p[0]; y = p[1]; }
+
+   void Set1w(const real_t x1, const real_t w) { x = x1; weight = w; }
+
+   void Set1w(const real_t *p) { x = p[0]; weight = p[1]; }
 };
 
 /// Class for an integration rule - an Array of IntegrationPoint.
@@ -121,6 +125,18 @@ private:
    void AddTriPoints3b(const int off, const real_t b, const real_t weight)
    { AddTriPoints3(off, (1. - b)/2., b, weight); }
 
+   void AddTriPoints3R(const int off, const real_t a, const real_t b,
+                       const real_t c, const real_t weight)
+   {
+      IntPoint(off + 0).Set2w(a, b, weight);
+      IntPoint(off + 1).Set2w(c, a, weight);
+      IntPoint(off + 2).Set2w(b, c, weight);
+   }
+
+   void AddTriPoints3R(const int off, const real_t a, const real_t b,
+                       const real_t weight)
+   { AddTriPoints3R(off, a, b, 1. - a - b, weight); }
+
    void AddTriPoints6(const int off, const real_t a, const real_t b,
                       const real_t c, const real_t weight)
    {
@@ -167,6 +183,14 @@ private:
       AddTetPoints3(off + 1, a, 1. - 3.*a, weight);
    }
 
+   // given b, add the permutations of (a,a,a,b), where 3*a + b = 1
+   void AddTetPoints4b(const int off, const real_t b, const real_t weight)
+   {
+      const real_t a = (1. - b)/3.;
+      IntPoint(off).Set(a, a, a, weight);
+      AddTetPoints3(off + 1, a, b, weight);
+   }
+
    // add the permutations of (a,a,b,b), 2*(a + b) = 1
    void AddTetPoints6(const int off, const real_t a, const real_t weight)
    {
@@ -185,37 +209,14 @@ private:
       AddTetPoints6(off + 6, a, bc, cb, weight);
    }
 
-   // add all 24 permutations of (a,b,c,d) where a+b+c+d = 1, all distinct
-   void AddTetPoints24(const int off, const real_t a, const real_t b,
-                       const real_t c, const real_t weight)
+   // given (b,c), add the permutations of (a,a,b,c), 2*a + b + c = 1
+   void AddTetPoints12bc(const int off, const real_t b, const real_t c,
+                         const real_t weight)
    {
-      const real_t d = 1. - a - b - c;
-      // all 24 permutations of 4 distinct barycentric coordinates
-      // permuting which coordinate goes to x, y, z (4th is 1-x-y-z)
-      IntPoint(off +  0).Set(a, b, c, weight);
-      IntPoint(off +  1).Set(a, b, d, weight);
-      IntPoint(off +  2).Set(a, c, b, weight);
-      IntPoint(off +  3).Set(a, c, d, weight);
-      IntPoint(off +  4).Set(a, d, b, weight);
-      IntPoint(off +  5).Set(a, d, c, weight);
-      IntPoint(off +  6).Set(b, a, c, weight);
-      IntPoint(off +  7).Set(b, a, d, weight);
-      IntPoint(off +  8).Set(b, c, a, weight);
-      IntPoint(off +  9).Set(b, c, d, weight);
-      IntPoint(off + 10).Set(b, d, a, weight);
-      IntPoint(off + 11).Set(b, d, c, weight);
-      IntPoint(off + 12).Set(c, a, b, weight);
-      IntPoint(off + 13).Set(c, a, d, weight);
-      IntPoint(off + 14).Set(c, b, a, weight);
-      IntPoint(off + 15).Set(c, b, d, weight);
-      IntPoint(off + 16).Set(c, d, a, weight);
-      IntPoint(off + 17).Set(c, d, b, weight);
-      IntPoint(off + 18).Set(d, a, b, weight);
-      IntPoint(off + 19).Set(d, a, c, weight);
-      IntPoint(off + 20).Set(d, b, a, weight);
-      IntPoint(off + 21).Set(d, b, c, weight);
-      IntPoint(off + 22).Set(d, c, a, weight);
-      IntPoint(off + 23).Set(d, c, b, weight);
+      const real_t a = (1. - b - c)/2.;
+      AddTetPoints3(off,     a, b, weight);
+      AddTetPoints3(off + 3, a, c, weight);
+      AddTetPoints6(off + 6, a, b, c, weight);
    }
 
 public:

fem/linearform.cpp

@@ -297,8 +297,7 @@ void LinearForm::Assemble()
          tr = mesh->GetBdrFaceTransformations(i);
          if (tr != NULL)
          {
-            mfem::DofTransformation doftrans;
-            fes -> GetElementVDofs (tr -> Elem1No, vdofs, doftrans);
+            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
             for (int k = 0; k < boundary_face_integs.Size(); k++)
             {
                if (boundary_face_integs_marker[k] &&
@@ -308,7 +307,6 @@ void LinearForm::Assemble()
                boundary_face_integs[k]->
                AssembleRHSElementVect(*fes->GetFE(tr->Elem1No),
                                       *tr, elemvect);
-               doftrans.TransformDual(elemvect);
                AddElementVector (vdofs, elemvect);
             }
          }

fem/lininteg.hpp

@@ -164,8 +164,8 @@ private:
 
 public:
    /// Constructs the domain integrator $ (Q, \nabla v) $
-   DomainLFGradIntegrator(VectorCoefficient &QF, const IntegrationRule *ir = NULL)
-      : DeltaLFIntegrator(QF, ir), Q(QF) { }
+   DomainLFGradIntegrator(VectorCoefficient &QF)
+      : DeltaLFIntegrator(QF), Q(QF) { }
 
    bool SupportsDevice() const override { return true; }
 

fem/pgridfunc.cpp

@@ -545,8 +545,6 @@ void ParGridFunction::GetElementDofValues(int el, Vector &dof_vals) const
 
 void ParGridFunction::ProjectCoefficient(Coefficient &coeff, ProjectType type)
 {
-   MFEM_VERIFY(VectorDim() == 1,
-               "Cannot project scalar coefficient onto vector ParGridFunction");
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
 
    if (delta_c == NULL)
@@ -717,8 +715,7 @@ void ParGridFunction::ProjectCoefficientElementL2(VectorCoefficient &vcoeff)
 }
 
 
-void ParGridFunction::ProjectDiscCoefficient(
-   std::variant<Coefficient*, VectorCoefficient*> coeff)
+void ParGridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff)
 {
    // local maximal element attribute for each dof
    Array<int> ldof_attr;
@@ -764,9 +761,6 @@ void ParGridFunction::ProjectDiscCoefficient(
 
 void ParGridFunction::ProjectDiscCoefficient(Coefficient &coeff, AvgType type)
 {
-   MFEM_VERIFY(
-      VectorDim() == 1,
-      "Cannot project scalar coefficient onto a vector ParGridFunction");
    // Harmonic  (x1 ... xn) = [ (1/x1 + ... + 1/xn) / n ]^-1.
    // Arithmetic(x1 ... xn) = (x1 + ... + xn) / n.
 
@@ -792,8 +786,6 @@ void ParGridFunction::ProjectDiscCoefficient(VectorCoefficient &vcoeff,
    // Harmonic  (x1 ... xn) = [ (1/x1 + ... + 1/xn) / n ]^-1.
    // Arithmetic(x1 ... xn) = (x1 + ... + xn) / n.
 
-   MFEM_VERIFY(VectorDim() == vcoeff.GetVDim(), "vcoeff vdim != VectorDim()");
-
    // Number of zones that contain a given dof.
    Array<int> zones_per_vdof;
    AccumulateAndCountZones(vcoeff, type, zones_per_vdof);
@@ -866,12 +858,6 @@ void ParGridFunction::ProjectBdrCoefficient(
 #endif
 }
 
-void ParGridFunction::ProjectBdrCoefficient(VectorCoefficient &vcoeff,
-                                            const Array<int> &attr)
-{
-   ProjectBdrCoefficient(NULL, &vcoeff, attr);
-}
-
 void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                                    const Array<int> &bdr_attr)
 {

fem/pgridfunc.hpp

@@ -63,12 +63,6 @@ protected:
    void ProjectBdrCoefficient(Coefficient *coeff[], VectorCoefficient *vcoeff,
                               const Array<int> &attr);
 
-   /** @brief Project a discontinuous (vector) coefficient as a grid function on
-       a continuous finite element space. The values in shared dofs are
-       determined from the element with maximal attribute. */
-   virtual void ProjectDiscCoefficient(
-      std::variant<Coefficient*, VectorCoefficient*> coeff) override;
-
 public:
    ParGridFunction() { pfes = NULL; }
 
@@ -274,6 +268,11 @@ public:
                            ProjectType type = ProjectType::DEFAULT) override;
 
    using GridFunction::ProjectDiscCoefficient;
+   /** @brief Project a discontinuous vector coefficient as a grid function on
+       a continuous finite element space. The values in shared dofs are
+       determined from the element with maximal attribute. */
+   void ProjectDiscCoefficient(VectorCoefficient &coeff) override;
+
    void ProjectDiscCoefficient(Coefficient &coeff, AvgType type) override;
 
    void ProjectDiscCoefficient(VectorCoefficient &vcoeff, AvgType type) override;
@@ -281,7 +280,8 @@ public:
    using GridFunction::ProjectBdrCoefficient;
 
    void ProjectBdrCoefficient(VectorCoefficient &vcoeff,
-                              const Array<int> &attr) override;
+                              const Array<int> &attr) override
+   { ProjectBdrCoefficient(NULL, &vcoeff, attr); }
 
    void ProjectBdrCoefficient(Coefficient *coeff[],
                               const Array<int> &attr) override

fem/prestriction.cpp

@@ -321,17 +321,12 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
    const int vd = vdim;
    const bool t = byvdim;
    const int threshold = ndofs;
-   const int nsdofs = pfes.GetFaceNbrVSize() / vd;
+   const int nsdofs = pfes.GetFaceNbrVSize();
    auto d_indices1 = scatter_indices1.Read();
    auto d_indices2 = scatter_indices2.Read();
    auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   const int ne_shared = nsdofs / elem_dofs;
-   const int nedof = elem_dofs;
-   // Note: the shape of face_nbr_data, as determined by
-   // ParFiniteElementSpace::ExchangeFaceNbrData, is (elem_dofs, vdim,
-   // ne_shared), independent of the ordering (byNODES or byVDIM) of the finite
-   // element space.
-   auto d_x_shared = Reshape(face_nbr_data.Read(), elem_dofs, vd, ne_shared);
+   auto d_x_shared = Reshape(face_nbr_data.Read(),
+                             t?vd:nsdofs, t?nsdofs:vd);
    auto d_y = Reshape(y.Write(), nface_dofs, vd, 2, nf);
    mfem::forall(nfdofs, [=] MFEM_HOST_DEVICE (int i)
    {
@@ -351,9 +346,8 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
          }
          else if (idx2>=threshold) // shared boundary
          {
-            const int e_shared = (idx2 - threshold) / nedof;
-            const int i_shared = (idx2 - threshold) % nedof;
-            d_y(dof, c, 1, face) = d_x_shared(i_shared,c,e_shared);
+            d_y(dof, c, 1, face) = d_x_shared(t?c:(idx2-threshold),
+                                              t?(idx2-threshold):c);
          }
          else // true boundary
          {

fem/restriction.cpp

@@ -1398,17 +1398,20 @@ void L2FaceRestriction::PermuteAndSetSharedFaceDofsScatterIndices2(
    const int dim = fes.GetMesh()->Dimension();
    const int dof1d = fes.GetTypicalFE()->GetOrder()+1;
    fes.GetTypicalFE()->GetFaceMap(face_id2, face_map);
+   Array<int> face_nbr_dofs;
+   const ParFiniteElementSpace &pfes =
+      static_cast<const ParFiniteElementSpace&>(this->fes);
+   pfes.GetFaceNbrElementVDofs(elem_index, face_nbr_dofs);
 
    for (int face_dof_elem1 = 0; face_dof_elem1 < face_dofs; ++face_dof_elem1)
    {
       const int face_dof_elem2 = PermuteFaceL2(dim, face_id1, face_id2,
                                                orientation, dof1d, face_dof_elem1);
       const int volume_dof_elem2 = face_map[face_dof_elem2];
-      // Encode the volume DOF index and element index
-      const int global_dof_elem2 = elem_index*elem_dofs + volume_dof_elem2;
+      const int global_dof_elem2 = face_nbr_dofs[volume_dof_elem2];
       const int restriction_dof_elem2 = face_dofs*face_index + face_dof_elem1;
       // Trick to differentiate dof location inter/shared
-      scatter_indices2[restriction_dof_elem2] = ndofs + global_dof_elem2;
+      scatter_indices2[restriction_dof_elem2] = ndofs+global_dof_elem2;
    }
 #endif
 }

general/array.cpp

@@ -23,7 +23,7 @@ namespace mfem
 template <class T>
 void Array<T>::Print(std::ostream &os, int width) const
 {
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       os << data[i];
       if ( !((i+1) % width) || i+1 == size )
@@ -44,7 +44,7 @@ void Array<T>::Save(std::ostream &os, int fmt) const
    {
       os << size << '\n';
    }
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       os << operator[](i) << '\n';
    }
@@ -55,11 +55,11 @@ void Array<T>::Load(std::istream &in, int fmt)
 {
    if (fmt == 0)
    {
-      int new_size;
+      bigint new_size;
       in >> new_size;
       SetSize(new_size);
    }
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       in >> operator[](i);
    }
@@ -71,7 +71,7 @@ T Array<T>::Max() const
    MFEM_ASSERT(size > 0, "Array is empty with size " << size);
 
    T max = operator[](0);
-   for (int i = 1; i < size; i++)
+   for (bigint i = 1; i < size; i++)
    {
       if (max < operator[](i))
       {
@@ -88,7 +88,7 @@ T Array<T>::Min() const
    MFEM_ASSERT(size > 0, "Array is empty with size " << size);
 
    T min = operator[](0);
-   for (int i = 1; i < size; i++)
+   for (bigint i = 1; i < size; i++)
    {
       if (operator[](i) < min)
       {
@@ -104,7 +104,7 @@ template <class T>
 void Array<T>::PartialSum()
 {
    T sum = static_cast<T>(0);
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       sum+=operator[](i);
       operator[](i) = sum;
@@ -116,9 +116,8 @@ void Array<T>::Abs()
 {
    static_assert(std::is_arithmetic<T>::value, "Use with arithmetic types!");
    const bool useDevice = UseDevice();
-   const int N = size;
    auto y = ReadWrite(useDevice);
-   mfem::forall_switch(useDevice, N, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(useDevice, size, [=] MFEM_HOST_DEVICE (bigint i)
    {
       y[i] = std::abs(y[i]);
    });
@@ -129,7 +128,7 @@ template <class T>
 T Array<T>::Sum() const
 {
    T sum = static_cast<T>(0);
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       sum+=operator[](i);
    }
@@ -141,7 +140,7 @@ template <class T>
 int Array<T>::IsSorted() const
 {
    T val_prev = operator[](0), val;
-   for (int i = 1; i < size; i++)
+   for (bigint i = 1; i < size; i++)
    {
       val=operator[](i);
       if (val < val_prev)
@@ -159,7 +158,7 @@ bool Array<T>::IsConstant() const
 {
    if (size < 2) { return true; }
    const T v0 = data[0];
-   for (int i = 1; i < size; i++)
+   for (bigint i = 1; i < size; i++)
    {
       if (data[i] != v0)
       {

general/array.hpp

@@ -50,9 +50,9 @@ protected:
    /// Pointer to data
    Memory<T> data;
    /// Size of the array
-   int size;
+   bigint size;
 
-   inline void GrowSize(int minsize);
+   inline void GrowSize(bigint minsize);
 
    static_assert(std::is_trivial<T>::value, "type T must be trivial");
 
@@ -68,11 +68,11 @@ public:
    inline Array(MemoryType mt) : data(mt), size(0) { }
 
    /// Creates array of @a asize elements
-   explicit inline Array(int asize)
+   explicit inline Array(bigint asize)
       : size(asize) { if (asize > 0) { data.New(asize); } }
 
    /// Creates array of @a asize elements with a given MemoryType
-   inline Array(int asize, MemoryType mt)
+   inline Array(bigint asize, MemoryType mt)
       : data(mt), size(asize) { if (asize > 0) { data.New(asize, mt); } }
 
    /** @brief Creates array using an externally allocated host pointer @a data_
@@ -81,7 +81,7 @@ public:
 
        When @a own_data is true, the pointer @a data_ must be allocated with
        MemoryType given by MemoryManager::GetHostMemoryType(). */
-   inline Array(T *data_, int asize, bool own_data = false)
+   inline Array(T *data_, bigint asize, bool own_data = false)
    { data.Wrap(data_, asize, own_data); size = asize; }
 
    /// Copy constructor: deep copy from @a src
@@ -93,7 +93,7 @@ public:
    inline Array(const Array<CT> &src);
 
    /// Construct an Array from a C-style array of static length
-   template <typename CT, int N>
+   template <typename CT, bigint N>
    explicit inline Array(const CT (&values)[N]);
 
    /// Construct an Array from a braced initializer list of convertible type
@@ -175,47 +175,47 @@ public:
    void MakeDataOwner() const { data.SetHostPtrOwner(true); }
 
    /// Return the logical size of the array.
-   inline int Size() const { return size; }
+   inline bigint Size() const { return size; }
 
    /// Change the logical size of the array, keep existing entries.
-   inline void SetSize(int nsize);
+   inline void SetSize(bigint nsize);
 
-   /// Same as SetSize(int) plus initialize new entries with 'initval'.
-   inline void SetSize(int nsize, const T &initval);
+   /// Same as SetSize(bigint) plus initialize new entries with 'initval'.
+   inline void SetSize(bigint nsize, const T &initval);
 
    /** @brief Resize the array to size @a nsize using MemoryType @a mt. Note
        that unlike the other versions of SetSize(), the current content of the
        array is not preserved. */
-   inline void SetSize(int nsize, MemoryType mt);
+   inline void SetSize(bigint nsize, MemoryType mt);
 
    /** Maximum number of entries the array can store without allocating more
        memory. */
-   inline int Capacity() const { return data.Capacity(); }
+   inline bigint Capacity() const { return data.Capacity(); }
 
    /// Ensures that the allocated size is at least the given size.
-   inline void Reserve(int capacity)
+   inline void Reserve(bigint capacity)
    { if (capacity > Capacity()) { GrowSize(capacity); } }
 
    /// Reference access to the ith element.
-   inline T & operator[](int i);
+   inline T & operator[](bigint i);
 
    /// Const reference access to the ith element.
-   inline const T &operator[](int i) const;
+   inline const T &operator[](bigint i) const;
 
    /// Append element 'el' to array, resize if necessary.
-   inline int Append(const T & el);
+   inline bigint Append(const T & el);
 
    /// STL-like push_back. Append element 'el' to array, resize if necessary.
    void push_back(const T &el) { Append(el); }
 
    /// Append another array to this array, resize if necessary.
-   inline int Append(const T *els, int nels);
+   inline bigint Append(const T *els, bigint nels);
 
    /// Append another array to this array, resize if necessary.
-   inline int Append(const Array<T> &els) { return Append(els, els.Size()); }
+   inline bigint Append(const Array<T> &els) { return Append(els, els.Size()); }
 
    /// Prepend an 'el' to the array, resize if necessary.
-   inline int Prepend(const T &el);
+   inline bigint Prepend(const T &el);
 
    /// Return the last element in the array.
    inline T &Last();
@@ -224,13 +224,13 @@ public:
    inline const T &Last() const;
 
    /// Append element when it is not yet in the array, return index.
-   inline int Union(const T & el);
+   inline bigint Union(const T & el);
 
    /// Return the first index where 'el' is found; return -1 if not found.
-   inline int Find(const T &el) const;
+   inline bigint Find(const T &el) const;
 
    /// Do bisection search for 'el' in a sorted array; return -1 if not found.
-   inline int FindSorted(const T &el) const;
+   inline bigint FindSorted(const T &el) const;
 
    /// Delete the last entry of the array.
    inline void DeleteLast() { if (size > 0) { size--; } }
@@ -253,18 +253,18 @@ public:
    /// Make this Array a reference to a pointer.
    /** When @a own_data is true, the pointer @a data_ must be allocated with
        MemoryType given by MemoryManager::GetHostMemoryType(). */
-   inline void MakeRef(T *data_, int size_, bool own_data = false);
+   inline void MakeRef(T *data_, bigint size_, bool own_data = false);
 
    /// Make this Array a reference to a pointer.
    /** When @a own_data is true, the pointer @a data_ must be allocated with
        MemoryType given by @a mt. */
-   inline void MakeRef(T *data_, int size, MemoryType mt, bool own_data);
+   inline void MakeRef(T *data_, bigint size, MemoryType mt, bool own_data);
 
    /// Make this Array a reference to 'master'.
    inline void MakeRef(const Array &master);
 
    /// Make this Array a reference to the given sub-Memory of @a base.
-   inline void MakeRef(Memory<T> &base, int offset, int size_);
+   inline void MakeRef(Memory<T> &base, bigint offset, bigint size_);
 
    /// Reset the Array to use the given external Memory @a mem and size @a s.
    /** If @a own_mem is false, the Array will not own any of the pointers of
@@ -273,7 +273,7 @@ public:
        Note that when @a own_mem is true, the @a mem object can be destroyed
        immediately by the caller but `mem.Delete()` should NOT be called since
        the Array object takes ownership of all pointers owned by @a mem. */
-   inline void NewMemoryAndSize(const Memory<T> &mem, int s, bool own_mem);
+   inline void NewMemoryAndSize(const Memory<T> &mem, bigint s, bool own_mem);
 
    /**
     * @brief Permute the array using the provided indices. Sorts the indices
@@ -289,7 +289,7 @@ public:
    inline void Permute(const I &indices) { Permute(I(indices)); }
 
    /// Copy sub array starting from @a offset out to the provided @a sa.
-   inline void GetSubArray(int offset, int sa_size, Array<T> &sa) const;
+   inline void GetSubArray(bigint offset, bigint sa_size, Array<T> &sa) const;
 
    /// Prints array to stream with width elements per row.
    void Print(std::ostream &out = mfem::out, int width = 4) const;
@@ -312,7 +312,7 @@ public:
 
    /** @brief Set the Array size to @a new_size and read that many entries from
        the stream @a in. */
-   void Load(int new_size, std::istream &in)
+   void Load(bigint new_size, std::istream &in)
    { SetSize(new_size); Load(in, 1); }
 
    /** @brief Find the maximal element in the array, using the comparison
@@ -335,7 +335,7 @@ public:
    void Unique()
    {
       T* end = std::unique((T*)data, data + size);
-      SetSize((int)(end - data));
+      SetSize((bigint)(end - data));
    }
 
    /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
@@ -421,7 +421,7 @@ template <class T>
 inline bool operator==(const Array<T> &LHS, const Array<T> &RHS)
 {
    if ( LHS.Size() != RHS.Size() ) { return false; }
-   for (int i=0; i<LHS.Size(); i++)
+   for (bigint i=0; i<LHS.Size(); i++)
    {
       if ( LHS[i] != RHS[i] ) { return false; }
    }
@@ -451,13 +451,13 @@ public:
    Array2D() { M = N = 0; }
 
    /// Construct an m x n 2D array.
-   Array2D(int m, int n) : array1d(m*n) { M = m; N = n; }
+   Array2D(int m, int n) : array1d(bigint(m)*n) { M = m; N = n; }
 
    Array2D(const Array2D &) = default;
    Array2D(Array2D &&) = default;
 
    /// Set the 2D array size to m x n.
-   void SetSize(int m, int n) { array1d.SetSize(m*n); M = m; N = n; }
+   void SetSize(int m, int n) { array1d.SetSize(bigint(m)*n); M = m; N = n; }
 
    int NumRows() const { return M; }
    int NumCols() const { return N; }
@@ -562,16 +562,16 @@ public:
 
    /// Construct a 3D array of size n1 x n2 x n3.
    Array3D(int n1, int n2, int n3)
-      : array1d(n1*n2*n3) { N2 = n2; N3 = n3; }
+      : array1d(bigint(n1)*n2*n3) { N2 = n2; N3 = n3; }
 
    /// Set the 3D array size to n1 x n2 x n3.
    void SetSize(int n1, int n2, int n3)
-   { array1d.SetSize(n1*n2*n3); N2 = n2; N3 = n3; }
+   { array1d.SetSize(bigint(n1)*n2*n3); N2 = n2; N3 = n3; }
 
    /// Get the 3D array size in the first dimension.
    int GetSize1() const
    {
-      const int size = array1d.Size();
+      const bigint size = array1d.Size();
       return size == 0 ? 0 : size / (N2 * N3);
    }
 
@@ -779,7 +779,7 @@ inline Array<T>::Array(const Array<CT> &src)
    : size(src.Size())
 {
    size > 0 ? data.New(size) : data.Reset();
-   for (int i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
+   for (bigint i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
 }
 
 template <typename T>
@@ -790,7 +790,7 @@ inline Array<T>::Array(std::initializer_list<CT> values) : Array(values.size())
    std::copy(values.begin(), values.end(), begin());
 }
 
-template <typename T> template <typename CT, int N>
+template <typename T> template <typename CT, bigint N>
 inline Array<T>::Array(const CT (&values)[N]) : Array(N)
 {
    std::copy(values, values + N, begin());
@@ -804,9 +804,9 @@ inline void Array<T>::Swap(Array &other)
 }
 
 template <class T>
-inline void Array<T>::GrowSize(int minsize)
+inline void Array<T>::GrowSize(bigint minsize)
 {
-   const int nsize = std::max(minsize, 2 * data.Capacity());
+   const bigint nsize = std::max(minsize, 2 * data.Capacity());
    Memory<T> p(nsize, data.GetMemoryType());
    p.CopyFrom(data, size);
    p.UseDevice(data.UseDevice());
@@ -829,7 +829,7 @@ template <typename T>
 template <typename I>
 inline void Array<T>::Permute(I &&indices)
 {
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       auto current = i;
       while (i != indices[current])
@@ -847,12 +847,12 @@ template <typename T> template <typename CT>
 inline Array<T> &Array<T>::operator=(const Array<CT> &src)
 {
    SetSize(src.Size());
-   for (int i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
+   for (bigint i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
    return *this;
 }
 
 template <class T>
-inline void Array<T>::SetSize(int nsize)
+inline void Array<T>::SetSize(bigint nsize)
 {
    MFEM_ASSERT( nsize>=0, "Size must be non-negative.  It is " << nsize );
    if (nsize > Capacity())
@@ -863,7 +863,7 @@ inline void Array<T>::SetSize(int nsize)
 }
 
 template <class T>
-inline void Array<T>::SetSize(int nsize, const T &initval)
+inline void Array<T>::SetSize(bigint nsize, const T &initval)
 {
    MFEM_ASSERT( nsize>=0, "Size must be non-negative.  It is " << nsize );
    if (nsize > size)
@@ -872,7 +872,7 @@ inline void Array<T>::SetSize(int nsize, const T &initval)
       {
          GrowSize(nsize);
       }
-      for (int i = size; i < nsize; i++)
+      for (bigint i = size; i < nsize; i++)
       {
          data[i] = initval;
       }
@@ -881,7 +881,7 @@ inline void Array<T>::SetSize(int nsize, const T &initval)
 }
 
 template <class T>
-inline void Array<T>::SetSize(int nsize, MemoryType mt)
+inline void Array<T>::SetSize(bigint nsize, MemoryType mt)
 {
    MFEM_ASSERT(nsize >= 0, "invalid new size: " << nsize);
    if (mt == data.GetMemoryType())
@@ -908,7 +908,7 @@ inline void Array<T>::SetSize(int nsize, MemoryType mt)
 }
 
 template <class T>
-inline T &Array<T>::operator[](int i)
+inline T &Array<T>::operator[](bigint i)
 {
    MFEM_ASSERT( i>=0 && i<size,
                 "Access element " << i << " of array, size = " << size );
@@ -916,7 +916,7 @@ inline T &Array<T>::operator[](int i)
 }
 
 template <class T>
-inline const T &Array<T>::operator[](int i) const
+inline const T &Array<T>::operator[](bigint i) const
 {
    MFEM_ASSERT( i>=0 && i<size,
                 "Access element " << i << " of array, size = " << size );
@@ -924,7 +924,7 @@ inline const T &Array<T>::operator[](int i) const
 }
 
 template <class T>
-inline int Array<T>::Append(const T &el)
+inline bigint Array<T>::Append(const T &el)
 {
    SetSize(size+1);
    data[size-1] = el;
@@ -932,12 +932,12 @@ inline int Array<T>::Append(const T &el)
 }
 
 template <class T>
-inline int Array<T>::Append(const T *els, int nels)
+inline bigint Array<T>::Append(const T *els, bigint nels)
 {
-   const int old_size = size;
+   const bigint old_size = size;
 
    SetSize(size + nels);
-   for (int i = 0; i < nels; i++)
+   for (bigint i = 0; i < nels; i++)
    {
       data[old_size+i] = els[i];
    }
@@ -945,10 +945,10 @@ inline int Array<T>::Append(const T *els, int nels)
 }
 
 template <class T>
-inline int Array<T>::Prepend(const T &el)
+inline bigint Array<T>::Prepend(const T &el)
 {
    SetSize(size+1);
-   for (int i = size-1; i > 0; i--)
+   for (bigint i = size-1; i > 0; i--)
    {
       data[i] = data[i-1];
    }
@@ -971,9 +971,9 @@ inline const T &Array<T>::Last() const
 }
 
 template <class T>
-inline int Array<T>::Union(const T &el)
+inline bigint Array<T>::Union(const T &el)
 {
-   int i = 0;
+   bigint i = 0;
    while ((i < size) && (data[i] != el)) { i++; }
    if (i == size)
    {
@@ -983,9 +983,9 @@ inline int Array<T>::Union(const T &el)
 }
 
 template <class T>
-inline int Array<T>::Find(const T &el) const
+inline bigint Array<T>::Find(const T &el) const
 {
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       if (data[i] == el) { return i; }
    }
@@ -993,18 +993,18 @@ inline int Array<T>::Find(const T &el) const
 }
 
 template <class T>
-inline int Array<T>::FindSorted(const T &el) const
+inline bigint Array<T>::FindSorted(const T &el) const
 {
    const T *begin = data, *end = begin + size;
    const T* first = std::lower_bound(begin, end, el);
    if (first == end || !(*first == el)) { return  -1; }
-   return (int)(first - begin);
+   return (bigint)(first - begin);
 }
 
 template <class T>
 inline void Array<T>::DeleteFirst(const T &el)
 {
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       if (data[i] == el)
       {
@@ -1027,8 +1027,8 @@ inline void Array<T>::DeleteAt(const Array<int> &indices)
    Array<int> sorted_indices(indices);
    sorted_indices.Sort();
 
-   int rm_count = 0;
-   for (int i = 0; i < size; i++)
+   bigint rm_count = 0;
+   for (bigint i = 0; i < size; i++)
    {
       if (rm_count < sorted_indices.Size() && i == sorted_indices[rm_count])
       {
@@ -1065,7 +1065,7 @@ inline void Array<T>::Copy(Array &copy) const
 }
 
 template <class T>
-inline void Array<T>::MakeRef(T *data_, int size_, bool own_data)
+inline void Array<T>::MakeRef(T *data_, bigint size_, bool own_data)
 {
    data.Delete();
    data.Wrap(data_, size_, own_data);
@@ -1073,7 +1073,8 @@ inline void Array<T>::MakeRef(T *data_, int size_, bool own_data)
 }
 
 template <class T>
-inline void Array<T>::MakeRef(T *data_, int size_, MemoryType mt, bool own_data)
+inline void Array<T>::MakeRef(T *data_, bigint size_, MemoryType mt,
+                              bool own_data)
 {
    data.Delete();
    data.Wrap(data_, size_, mt, own_data);
@@ -1089,7 +1090,7 @@ inline void Array<T>::MakeRef(const Array &master)
 }
 
 template <class T>
-inline void Array<T>::MakeRef(Memory<T> &base, int offset, int size_)
+inline void Array<T>::MakeRef(Memory<T> &base, bigint offset, bigint size_)
 {
    data.Delete();
    size = size_;
@@ -1098,7 +1099,7 @@ inline void Array<T>::MakeRef(Memory<T> &base, int offset, int size_)
 
 template <class T>
 inline void Array<T>::NewMemoryAndSize(
-   const Memory<T> &mem, int s, bool own_mem)
+   const Memory<T> &mem, bigint s, bool own_mem)
 {
    data.Delete();
    size = s;
@@ -1113,10 +1114,11 @@ inline void Array<T>::NewMemoryAndSize(
 }
 
 template <class T>
-inline void Array<T>::GetSubArray(int offset, int sa_size, Array<T> &sa) const
+inline void Array<T>::GetSubArray(bigint offset, bigint sa_size,
+                                  Array<T> &sa) const
 {
    sa.SetSize(sa_size);
-   for (int i = 0; i < sa_size; i++)
+   for (bigint i = 0; i < sa_size; i++)
    {
       sa[i] = (*this)[offset+i];
    }
@@ -1125,7 +1127,7 @@ inline void Array<T>::GetSubArray(int offset, int sa_size, Array<T> &sa) const
 template <class T>
 inline void Array<T>::operator=(const T &a)
 {
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       data[i] = a;
    }

general/arrays_by_name.hpp

@@ -14,7 +14,6 @@
 
 #include "../config/config.hpp"
 #include "array.hpp"
-#include "text.hpp"
 
 #include <iostream>
 #include <map>
@@ -248,8 +247,7 @@ inline void ArraysByName<T>::Print(std::ostream &os, int width) const
    os << data.size() << '\n';
    for (auto const &it : data)
    {
-      // Note: The method Load() can read any string formatted with std::quoted.
-      os << std::quoted(it.first) << '\n' << it.second.Size() << '\n';
+      os << '"' << it.first << '"' << '\n' << it.second.Size() << '\n';
       it.second.Print(os, width > 0 ? width : it.second.Size());
    }
 }
@@ -260,36 +258,31 @@ void ArraysByName<T>::Load(std::istream &in)
    int NumArrays;
    in >> NumArrays;
 
-   for (int i = 0; i < NumArrays; i++)
+   std::string ArrayLine, ArrayName;
+   for (int i=0; i < NumArrays; i++)
    {
       in >> std::ws;
-      // Read the name:
-      // - If the stream 'in' starts with " then parse it with the function
-      //   parse_quoted_string() from text.hpp. In this case, the name can be
-      //   empty. Note: this case allows for reading any string formatted using
-      //   std::quoted, e.g. as in the method Print().
-      // - If the name does not start with " then the name ends with the first
-      //   white space character (and the white space character is not included
-      //   in the name). Since white space characters are skipped before reading
-      //   the name, there will be at least one non-white-space character in the
-      //   name in this case.
-      std::string ArrayName;
-      if (in.peek() == '"')
+      getline(in, ArrayLine);
+
+      std::size_t q0 = ArrayLine.find('"');
+      std::size_t q1 = ArrayLine.rfind('"');
+
+      if (q0 != std::string::npos && q1 > q0)
       {
-         if (parse_quoted_string(ArrayName, in) != 0)
-         {
-            MFEM_ABORT("error parsing input!");
-         }
+         // Locate set name between first and last double quote
+         ArrayName = ArrayLine.substr(q0+1,q1-q0-1);
       }
       else
       {
-         in >> ArrayName;
-         MFEM_VERIFY(in.good(), "error parsing input!");
+         // If no double quotes found locate set name using white space
+         q1 = ArrayLine.find(' ');
+         ArrayName = ArrayLine.substr(0,q1-1);
       }
 
-      // Read the array
-      data[ArrayName].Load(in);
+      // Ignore the remainder of the line which may contain explanatory comments
+      data[ArrayName].Load(in, 0);
    }
+
 }
 
 }

general/device.cpp

@@ -24,6 +24,18 @@
 #include <map>
 #include <sstream>
 #include <iomanip>
+#if defined(__linux__)
+#include <sys/resource.h>  // getrusage
+#include <unistd.h>        // sysconf
+#include <cstdio>          // fopen, fscanf, fclose
+#elif defined(__APPLE__)
+#include <mach/mach_init.h>  // mach_task_self
+#include <mach/task.h>       // task_info
+#elif defined(_WIN32)
+#include <windows.h>
+#include <psapi.h>  // GetProcessMemoryInfo
+#pragma comment(lib, "psapi.lib")
+#endif
 
 namespace mfem
 {
@@ -718,6 +730,52 @@ void Device::DeviceMem(size_t *free, size_t *total)
 #endif
 }
 
+// static method
+void Device::HostMem(size_t *rss_p, size_t *maxrss_p)
+{
+   size_t rss = 0, maxrss = 0;
+
+#if defined(__linux__)
+   struct rusage usage;
+   if (getrusage(RUSAGE_SELF, &usage)) { usage.ru_maxrss = 0; }
+   maxrss = 1024*usage.ru_maxrss;
+   static const long PAGE_SIZE = sysconf(_SC_PAGESIZE);
+   FILE *statm = fopen("/proc/self/statm", "r");
+   if (statm)
+   {
+      // Values are measured in pages, see Table 1-3 at
+      // https://www.kernel.org/doc/Documentation/filesystems/proc.txt
+      long rss_pages;
+      if (fscanf(statm, "%*d %ld", &rss_pages) == EOF) { rss_pages = 0; }
+      fclose(statm);
+      rss = rss_pages * PAGE_SIZE;
+   }
+#elif defined(__APPLE__)
+   struct mach_task_basic_info info;
+   mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT;
+   if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO,
+                 (task_info_t)&info, &count) == KERN_SUCCESS)
+   {
+      rss = info.resident_size;
+      maxrss = info.resident_size_max;
+   }
+#elif defined(_WIN32)
+   PROCESS_MEMORY_COUNTERS mem_counters;
+   if (GetProcessMemoryInfo(GetCurrentProcess(),
+                            &mem_counters,
+                            sizeof(mem_counters)))
+   {
+      // Reference:
+      // https://learn.microsoft.com/en-us/windows/win32/api/psapi/ns-psapi-process_memory_counters
+      rss = mem_counters.WorkingSetSize;
+      maxrss = mem_counters.PeakWorkingSetSize;
+   }
+#endif
+
+   *rss_p = rss;
+   *maxrss_p = maxrss;
+}
+
 std::string Device::GetUUID(const int device_id)
 {
    std::stringstream res;
@@ -726,16 +784,16 @@ std::string Device::GetUUID(const int device_id)
    MFEM_GPU_CHECK(cudaGetDeviceProperties(&prop, device_id));
    for (int i = 0; i < 16; ++i)
    {
-      const unsigned b = static_cast<unsigned char>(prop.uuid.bytes[i]);
-      res << std::setfill('0') << std::setw(2) << std::hex << b;
+      res << std::setfill('0') << std::setw(2) << std::hex
+          << static_cast<unsigned>(prop.uuid.bytes[i]);
    }
 #elif defined(MFEM_USE_HIP)
    hipUUID uuid;
    MFEM_GPU_CHECK(hipDeviceGetUuid(&uuid, device_id));
    for (int i = 0; i < 16; ++i)
    {
-      const unsigned b = static_cast<unsigned char>(uuid.bytes[i]);
-      res << std::setfill('0') << std::setw(2) << std::hex << b;
+      res << std::setfill('0') << std::setw(2) << std::hex
+          << static_cast<unsigned>(uuid.bytes[i]);
    }
 #endif
    return res.str();

general/device.hpp

@@ -321,6 +321,10 @@ public:
 
    /** @brief Gets the @a free and @a total memory on the device. */
    static void DeviceMem(size_t *free, size_t *total);
+
+   /** @brief Gets the @a rss (resident set size) and @a maxrss (maximum
+       resident set size) memory of the host process, in bytes. */
+   static void HostMem(size_t *rss, size_t *maxrss);
 };
 
 
@@ -349,14 +353,14 @@ inline MemoryClass GetMemoryClass(const Memory<T> &mem, bool on_dev)
     HostMemoryClass, otherwise. */
 /** Also, if @a on_dev = true, the device flag of @a mem will be set. */
 template <typename T>
-inline const T *Read(const Memory<T> &mem, int size, bool on_dev = true)
+inline const T *Read(const Memory<T> &mem, bigint size, bool on_dev = true)
 {
    return mem.Read(GetMemoryClass(mem, on_dev), size);
 }
 
-/** @brief Shortcut to Read(const Memory<T> &mem, int size, false) */
+/** @brief Shortcut to Read(const Memory<T> &mem, bigint size, false) */
 template <typename T>
-inline const T *HostRead(const Memory<T> &mem, int size)
+inline const T *HostRead(const Memory<T> &mem, bigint size)
 {
    return mfem::Read(mem, size, false);
 }
@@ -366,14 +370,14 @@ inline const T *HostRead(const Memory<T> &mem, int size)
     HostMemoryClass, otherwise. */
 /** Also, if @a on_dev = true, the device flag of @a mem will be set. */
 template <typename T>
-inline T *Write(Memory<T> &mem, int size, bool on_dev = true)
+inline T *Write(Memory<T> &mem, bigint size, bool on_dev = true)
 {
    return mem.Write(GetMemoryClass(mem, on_dev), size);
 }
 
-/** @brief Shortcut to Write(const Memory<T> &mem, int size, false) */
+/** @brief Shortcut to Write(const Memory<T> &mem, bigint size, false) */
 template <typename T>
-inline T *HostWrite(Memory<T> &mem, int size)
+inline T *HostWrite(Memory<T> &mem, bigint size)
 {
    return mfem::Write(mem, size, false);
 }
@@ -383,14 +387,14 @@ inline T *HostWrite(Memory<T> &mem, int size)
     HostMemoryClass, otherwise. */
 /** Also, if @a on_dev = true, the device flag of @a mem will be set. */
 template <typename T>
-inline T *ReadWrite(Memory<T> &mem, int size, bool on_dev = true)
+inline T *ReadWrite(Memory<T> &mem, bigint size, bool on_dev = true)
 {
    return mem.ReadWrite(GetMemoryClass(mem, on_dev), size);
 }
 
-/** @brief Shortcut to ReadWrite(Memory<T> &mem, int size, false) */
+/** @brief Shortcut to ReadWrite(Memory<T> &mem, bigint size, false) */
 template <typename T>
-inline T *HostReadWrite(Memory<T> &mem, int size)
+inline T *HostReadWrite(Memory<T> &mem, bigint size)
 {
    return mfem::ReadWrite(mem, size, false);
 }

general/error.hpp

@@ -68,7 +68,8 @@ void mfem_error(const char *msg = NULL);
 __attribute__((enzyme_inactive))
 #endif
 void mfem_warning(const char *msg = NULL);
-}
+
+} // namespace mfem
 
 #ifndef _MFEM_FUNC_NAME
 #ifndef _MSC_VER

general/forall.hpp

@@ -176,10 +176,10 @@ private:
 // with CUDA/HIP language. Otherwise, this macro is a no-op.
 #if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 #define MFEM_GPU_FORALL(i, N,...) CuWrap1D(N, [=] MFEM_DEVICE      \
-                                       (int i) {__VA_ARGS__})
+                                       (bigint i) {__VA_ARGS__})
 #elif defined(MFEM_USE_HIP) && defined(__HIP__)
 #define MFEM_GPU_FORALL(i, N,...) HipWrap1D(N, [=] MFEM_DEVICE     \
-                                        (int i) {__VA_ARGS__})
+                                        (bigint i) {__VA_ARGS__})
 #else
 #define MFEM_GPU_FORALL(i, N,...) do { } while (false)
 #endif
@@ -189,7 +189,7 @@ private:
 
 // The MFEM_FORALL wrapper
 #define MFEM_FORALL(i,N,...) \
-   ForallWrap<1>(true,N,[=] MFEM_HOST_DEVICE (int i) {__VA_ARGS__})
+   ForallWrap<1>(true,N,[=] MFEM_HOST_DEVICE (bigint i) {__VA_ARGS__})
 
 // MFEM_FORALL with a 2D CUDA block
 #define MFEM_FORALL_2D(i,N,X,Y,BZ,...) \
@@ -208,16 +208,16 @@ private:
 // example the functions in vector.cpp, where we don't want to use the mfem
 // device for operations on small vectors.
 #define MFEM_FORALL_SWITCH(use_dev,i,N,...) \
-   ForallWrap<1>(use_dev,N,[=] MFEM_HOST_DEVICE (int i) {__VA_ARGS__})
+   ForallWrap<1>(use_dev,N,[=] MFEM_HOST_DEVICE (bigint i) {__VA_ARGS__})
 
 
 /// OpenMP backend
 template <typename HBODY>
-void OmpWrap(const int N, HBODY &&h_body)
+void OmpWrap(const bigint N, HBODY &&h_body)
 {
 #ifdef MFEM_USE_OPENMP
    #pragma omp parallel for
-   for (int k = 0; k < N; k++)
+   for (bigint k = 0; k < N; k++)
    {
       h_body(k);
    }
@@ -296,7 +296,7 @@ using hip_threads_z =
 
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_CUDA) && defined(__CUDACC__)
 template <typename DBODY>
-void RajaCuWrap1D(const int N, DBODY &&d_body)
+void RajaCuWrap1D(const bigint N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
    RAJA::forall<RAJA::cuda_exec<MFEM_CUDA_BLOCKS,true>>(RAJA::RangeSegment(0,N),
@@ -364,7 +364,7 @@ template <>
 struct RajaCuWrap<1>
 {
    template <typename DBODY>
-   static void run(const int N, DBODY &&d_body,
+   static void run(const bigint N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
       RajaCuWrap1D(N, d_body);
@@ -397,7 +397,7 @@ struct RajaCuWrap<3>
 
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_HIP) && defined(__HIP__)
 template <typename DBODY>
-void RajaHipWrap1D(const int N, DBODY &&d_body)
+void RajaHipWrap1D(const bigint N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
    RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true>>(RAJA::RangeSegment(0,N),
@@ -465,7 +465,7 @@ template <>
 struct RajaHipWrap<1>
 {
    template <typename DBODY>
-   static void run(const int N, DBODY &&d_body,
+   static void run(const bigint N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
       RajaHipWrap1D(N, d_body);
@@ -500,7 +500,7 @@ struct RajaHipWrap<3>
 #if defined(MFEM_USE_RAJA) && defined(RAJA_ENABLE_OPENMP)
 
 template <typename HBODY>
-void RajaOmpWrap(const int N, HBODY &&h_body)
+void RajaOmpWrap(const bigint N, HBODY &&h_body)
 {
    RAJA::forall<RAJA::omp_parallel_for_exec>(RAJA::RangeSegment(0,N), h_body);
 }
@@ -546,7 +546,7 @@ void RajaOmpWrap3D(const int Nx, const int Ny, const int Nz, HBODY &&h_body)
 
 /// RAJA sequential loop backend
 template <typename HBODY>
-void RajaSeqWrap(const int N, HBODY &&h_body)
+void RajaSeqWrap(const bigint N, HBODY &&h_body)
 {
 #ifdef MFEM_USE_RAJA
 
@@ -571,9 +571,9 @@ void RajaSeqWrap(const int N, HBODY &&h_body)
 #if defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 
 template <typename BODY> __global__ static
-void CuKernel1D(const int N, BODY body)
+void CuKernel1D(const bigint N, BODY body)
 {
-   const int k = blockDim.x*blockIdx.x + threadIdx.x;
+   const bigint k = bigint(blockDim.x)*blockIdx.x + threadIdx.x;
    if (k >= N) { return; }
    body(k);
 }
@@ -612,10 +612,10 @@ static void CuKernel3DLaunchBounds(const int N, BODY body)
 }
 
 template <const int BLCK = MFEM_CUDA_BLOCKS, typename DBODY>
-void CuWrap1D(const int N, DBODY &&d_body)
+void CuWrap1D(const bigint N, DBODY &&d_body)
 {
    if (N==0) { return; }
-   const int GRID = (N+BLCK-1)/BLCK;
+   const unsigned int GRID = (N+BLCK-1)/BLCK;
    CuKernel1D<<<GRID,BLCK>>>(N, d_body);
    MFEM_GPU_CHECK(cudaGetLastError());
 }
@@ -676,7 +676,7 @@ template <int MAX_THREADS_PER_BLOCK>
 struct CuWrap<1, MAX_THREADS_PER_BLOCK>
 {
    template <typename DBODY>
-   static void run(const int N, DBODY &&d_body,
+   static void run(const bigint N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
       CuWrap1D<MFEM_CUDA_BLOCKS>(N, d_body);
@@ -735,9 +735,9 @@ struct CuWrap<3, MAX_THREADS_PER_BLOCK>
 #if defined(MFEM_USE_HIP) && defined(__HIP__)
 
 template <typename BODY> __global__ static
-void HipKernel1D(const int N, BODY body)
+void HipKernel1D(const bigint N, BODY body)
 {
-   const int k = hipBlockDim_x*hipBlockIdx_x + hipThreadIdx_x;
+   const bigint k = bigint(hipBlockDim_x)*hipBlockIdx_x + hipThreadIdx_x;
    if (k >= N) { return; }
    body(k);
 }
@@ -775,10 +775,10 @@ static void HipKernel3DLaunchBounds(const int N, BODY body)
 }
 
 template <int BLCK = MFEM_HIP_BLOCKS, typename DBODY>
-void HipWrap1D(const int N, DBODY &&d_body)
+void HipWrap1D(const bigint N, DBODY &&d_body)
 {
    if (N==0) { return; }
-   const int GRID = (N+BLCK-1)/BLCK;
+   const unsigned int GRID = (N+BLCK-1)/BLCK;
    hipLaunchKernelGGL(HipKernel1D,GRID,BLCK,0,nullptr,N,d_body);
    MFEM_GPU_CHECK(hipGetLastError());
 }
@@ -839,7 +839,7 @@ template <int MAX_THREADS_PER_BLOCK>
 struct HipWrap<1, MAX_THREADS_PER_BLOCK>
 {
    template <typename DBODY>
-   static void run(const int N, DBODY &&d_body,
+   static void run(const bigint N, DBODY &&d_body,
                    const int X, const int Y, const int Z, const int G)
    {
       HipWrap1D<MFEM_HIP_BLOCKS>(N, d_body);
@@ -897,7 +897,7 @@ struct HipWrap<3, MAX_THREADS_PER_BLOCK>
 /// Forall host & device kernel dispatch
 template <int DIM, int MAX_THREADS_PER_BLOCK = 0,
           typename d_lambda, typename h_lambda>
-inline void ForallWrap(const bool use_dev, const int N,
+inline void ForallWrap(const bool use_dev, const bigint N,
                        d_lambda &&d_body, h_lambda &&h_body,
                        const int X=0, const int Y=0, const int Z=0,
                        const int G=0)
@@ -963,13 +963,13 @@ backend_cpu:
    // Handle Backend::CPU. This is also a fallback for any allowed backends not
    // handled above, e.g. OCCA_CPU with configuration 'occa-cpu,cpu', or
    // OCCA_OMP with configuration 'occa-omp,cpu'.
-   for (int k = 0; k < N; k++) { h_body(k); }
+   for (bigint k = 0; k < N; k++) { h_body(k); }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 /// Forall host & device kernel wrappers
 template <int DIM, typename lambda>
-inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
+inline void ForallWrap(const bool use_dev, const bigint N, lambda &&body,
                        const int X=0, const int Y=0, const int Z=0,
                        const int G=0)
 {
@@ -977,7 +977,7 @@ inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
 }
 
 template <int DIM, int MAX_THREADS_PER_BLOCK, typename lambda>
-inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
+inline void ForallWrap(const bool use_dev, const bigint N, lambda &&body,
                        const int X=0, const int Y=0, const int Z=0,
                        const int G=0)
 {
@@ -987,14 +987,14 @@ inline void ForallWrap(const bool use_dev, const int N, lambda &&body,
 ///////////////////////////////////////////////////////////////////////////////
 // forall interfaces
 template<typename lambda>
-inline void forall(int N, lambda &&body) { ForallWrap<1>(true, N, body); }
+inline void forall(bigint N, lambda &&body) { ForallWrap<1>(true, N, body); }
 
 template<typename lambda>
 inline void forall(int Nx, int Ny, lambda &&body)
 {
    if (Device::Allows(Backend::DEVICE_MASK))
    {
-      mfem::forall(Nx * Ny, [=] MFEM_HOST_DEVICE(int idx)
+      mfem::forall(bigint(Nx) * Ny, [=] MFEM_HOST_DEVICE(bigint idx)
       {
          int j = idx / Nx;
          int i = idx % Nx;
@@ -1030,12 +1030,12 @@ inline void forall(int Nx, int Ny, int Nz, lambda &&body)
 {
    if (Device::Allows(Backend::DEVICE_MASK))
    {
-      mfem::forall(Nx * Ny * Nz, [=] MFEM_HOST_DEVICE(int idx)
+      mfem::forall(bigint(Nx) * Ny * Nz, [=] MFEM_HOST_DEVICE(bigint idx)
       {
          int i = idx % Nx;
-         int j = idx / Nx;
-         int k = j / Ny;
-         j = j % Ny;
+         bigint jk = idx / Nx;
+         int k = jk / Ny;
+         int j = jk % Ny;
          body(i, j, k);
       });
    }
@@ -1067,7 +1067,7 @@ inline void forall(int Nx, int Ny, int Nz, lambda &&body)
 }
 
 template<typename lambda>
-inline void forall_switch(bool use_dev, int N, lambda &&body)
+inline void forall_switch(bool use_dev, bigint N, lambda &&body)
 {
    ForallWrap<1>(use_dev, N, body);
 }
@@ -1090,12 +1090,6 @@ inline void forall_2D_batch(int N, int X, int Y, int BZ, lambda &&body)
    ForallWrap<2>(true, N, body, X, Y, BZ);
 }
 
-template<int MAX_THREADS_PER_BLOCK, typename lambda>
-inline void forall_2D_batch(int N, int X, int Y, int BZ, lambda &&body)
-{
-   ForallWrap<2, MAX_THREADS_PER_BLOCK>(true, N, body, X, Y, BZ);
-}
-
 template<typename lambda>
 inline void forall_3D(int N, int X, int Y, int Z, lambda &&body)
 {

general/mem_manager.cpp

@@ -150,8 +150,8 @@ template void Memory<int>::PrintFlags() const;
 template void Memory<real_t>::PrintFlags() const;
 
 // Instantiate Memory<T>::CompareHostAndDevice for T = int and T = real_t.
-template int Memory<int>::CompareHostAndDevice(int size) const;
-template int Memory<real_t>::CompareHostAndDevice(int size) const;
+template int Memory<int>::CompareHostAndDevice(bigint size) const;
+template int Memory<real_t>::CompareHostAndDevice(bigint size) const;
 
 
 namespace internal

general/mem_manager.hpp

@@ -197,7 +197,7 @@ protected:
    /** The type of the pointer is given by the field #h_mt; it can be any type
        from MemoryClass::HOST. */
    T *h_ptr;
-   int capacity; ///< Size of the allocated memory
+   bigint capacity; ///< Size of the allocated memory
    MemoryType h_mt; ///< Host memory type
    mutable unsigned flags; ///< Bit flags defined from the #FlagMask enum
    // 'flags' is mutable so that it can be modified in Set{Host,Device}PtrOwner,
@@ -236,7 +236,7 @@ public:
    /// Allocate host memory for @a size entries.
    /** The allocation uses the current host memory type returned by
        MemoryManager::GetHostMemoryType(). */
-   explicit Memory(int size) { New(size); }
+   explicit Memory(bigint size) { New(size); }
 
    /// Creates a new empty Memory object with host MemoryType @a mt.
    explicit Memory(MemoryType mt) { Reset(mt); }
@@ -245,19 +245,20 @@ public:
        @a mt. */
    /** The newly allocated memory is not initialized, however the given
        MemoryType is still set as valid. */
-   Memory(int size, MemoryType mt) { New(size, mt); }
+   Memory(bigint size, MemoryType mt) { New(size, mt); }
 
    /** @brief Allocate memory for @a size entries with the given host MemoryType
        @a h_mt and device MemoryType @a d_mt. */
    /** The newly allocated memory is not initialized. The host pointer is set as
        valid. */
-   Memory(int size, MemoryType h_mt, MemoryType d_mt) { New(size, h_mt, d_mt); }
+   Memory(bigint size, MemoryType h_mt, MemoryType d_mt)
+   { New(size, h_mt, d_mt); }
 
    /** @brief Wrap an externally allocated host pointer, @a ptr with the current
        host memory type returned by MemoryManager::GetHostMemoryType(). */
    /** The parameter @a own determines whether @a ptr will be deleted when the
        method Delete() is called. */
-   explicit Memory(T *ptr, int size, bool own) { Wrap(ptr, size, own); }
+   explicit Memory(T *ptr, bigint size, bool own) { Wrap(ptr, size, own); }
 
    /// Wrap an externally allocated pointer, @a ptr, of the given MemoryType.
    /** The new memory object will have the given MemoryType set as valid.
@@ -267,13 +268,13 @@ public:
 
        The parameter @a own determines whether @a ptr will be deleted when the
        method Delete() is called. */
-   Memory(T *ptr, int size, MemoryType mt, bool own)
+   Memory(T *ptr, bigint size, MemoryType mt, bool own)
    { Wrap(ptr, size, mt, own); }
 
    /** @brief Alias constructor. Create a Memory object that points inside the
        Memory object @a base. */
    /** The new Memory object uses the same MemoryType(s) as @a base. */
-   Memory(const Memory &base, int offset, int size)
+   Memory(const Memory &base, bigint offset, bigint size)
    { MakeAlias(base, offset, size); }
 
    /// Destructor: default.
@@ -319,7 +320,7 @@ public:
    { flags = use_dev ? (flags | USE_DEVICE) : (flags & ~USE_DEVICE); }
 
    /// Return the size of the allocated memory.
-   int Capacity() const { return capacity; }
+   bigint Capacity() const { return capacity; }
 
    /// Reset the memory to be empty, ensuring that Delete() will be a no-op.
    /** This is the Memory class equivalent to setting a pointer to NULL, see
@@ -339,7 +340,7 @@ public:
    /** @brief Allocate host memory for @a size entries with the current host
        memory type returned by MemoryManager::GetHostMemoryType(). */
    /** @note The current memory is NOT deleted by this method. */
-   inline void New(int size);
+   inline void New(bigint size);
 
    /// Allocate memory for @a size entries with the given MemoryType.
    /** The newly allocated memory is not initialized, however the given
@@ -353,7 +354,7 @@ public:
        to be the dual of @a mt, see MemoryManager::GetDualMemoryType().
 
        @note The current memory is NOT deleted by this method. */
-   inline void New(int size, MemoryType mt);
+   inline void New(bigint size, MemoryType mt);
 
    /** @brief Allocate memory for @a size entries with the given host MemoryType
        @a h_mt and device MemoryType @a d_mt. */
@@ -361,7 +362,7 @@ public:
        valid.
 
        @note The current memory is NOT deleted by this method. */
-   inline void New(int size, MemoryType h_mt, MemoryType d_mt);
+   inline void New(bigint size, MemoryType h_mt, MemoryType d_mt);
 
    /** @brief Wrap an externally allocated host pointer, @a ptr with the current
        host memory type returned by MemoryManager::GetHostMemoryType(). */
@@ -369,7 +370,7 @@ public:
        method Delete() is called.
 
        @note The current memory is NOT deleted by this method. */
-   inline void Wrap(T *ptr, int size, bool own);
+   inline void Wrap(T *ptr, bigint size, bool own);
 
    /// Wrap an externally allocated pointer, @a ptr, of the given MemoryType.
    /** The new memory object will have the given MemoryType set as valid.
@@ -381,7 +382,7 @@ public:
        method Delete() is called.
 
        @note The current memory is NOT deleted by this method. */
-   inline void Wrap(T *ptr, int size, MemoryType mt, bool own);
+   inline void Wrap(T *ptr, bigint size, MemoryType mt, bool own);
 
    /** Wrap an externally pair of allocated pointers, @a h_ptr and @a d_ptr,
        of the given host MemoryType @a h_mt. */
@@ -405,14 +406,14 @@ public:
          - SetDevicePtrOwner.
 
        @note The current memory is NOT deleted by this method. */
-   inline void Wrap(T *h_ptr, T *d_ptr, int size, MemoryType h_mt, bool own,
+   inline void Wrap(T *h_ptr, T *d_ptr, bigint size, MemoryType h_mt, bool own,
                     bool valid_host = false, bool valid_device = true);
 
    /// Create a memory object that points inside the memory object @a base.
    /** The new Memory object uses the same MemoryType(s) as @a base.
 
        @note The current memory is NOT deleted by this method. */
-   inline void MakeAlias(const Memory &base, int offset, int size);
+   inline void MakeAlias(const Memory &base, bigint offset, bigint size);
 
    /// Set the device MemoryType to be used by the Memory object.
    /** If the specified @a d_mt is not a device MemoryType, i.e. not one of the
@@ -437,10 +438,10 @@ public:
    inline void DeleteDevice(bool copy_to_host = true);
 
    /// Array subscript operator for host memory.
-   inline T &operator[](int idx);
+   inline T &operator[](bigint idx);
 
    /// Array subscript operator for host memory, const version.
-   inline const T &operator[](int idx) const;
+   inline const T &operator[](bigint idx) const;
 
    /// Direct access to the host memory as T* (implicit conversion).
    /** When the type T is const-qualified, this method can be used only if the
@@ -492,11 +493,11 @@ public:
        Read() or Write() should be used instead of this method.
 
        The parameter @a size must not exceed the Capacity(). */
-   inline T *ReadWrite(MemoryClass mc, int size);
+   inline T *ReadWrite(MemoryClass mc, bigint size);
 
    /// Get read-only access to the memory with the given MemoryClass.
    /** The parameter @a size must not exceed the Capacity(). */
-   inline const T *Read(MemoryClass mc, int size) const;
+   inline const T *Read(MemoryClass mc, bigint size) const;
 
    /// Get write-only access to the memory with the given MemoryClass.
    /** The parameter @a size must not exceed the Capacity().
@@ -504,7 +505,7 @@ public:
        The contents of the returned pointer is undefined, unless it was
        validated by a previous call to Read() or ReadWrite() with
        the same MemoryClass. */
-   inline T *Write(MemoryClass mc, int size);
+   inline T *Write(MemoryClass mc, bigint size);
 
    /// Copy the host/device pointer validity flags from @a other to @a *this.
    /** This method synchronizes the pointer validity flags of two Memory objects
@@ -521,7 +522,7 @@ public:
        of the base incorrect. Calling this method will ensure that @a base is
        up-to-date. Note that this is achieved by moving/copying @a *this (if
        necessary), and not @a base. */
-   inline void SyncAlias(const Memory &base, int alias_size) const;
+   inline void SyncAlias(const Memory &base, bigint alias_size) const;
 
    /** @brief Return a MemoryType that is currently valid. If both the host and
        the device pointers are currently valid, then the device memory type is
@@ -544,20 +545,20 @@ public:
    /// Copy @a size entries from @a src to @a *this.
    /** The given @a size should not exceed the Capacity() of the source @a src
        and the destination, @a *this. */
-   inline void CopyFrom(const Memory &src, int size);
+   inline void CopyFrom(const Memory &src, bigint size);
 
    /// Copy @a size entries from the host pointer @a src to @a *this.
    /** The given @a size should not exceed the Capacity() of @a *this. */
-   inline void CopyFromHost(const T *src, int size);
+   inline void CopyFromHost(const T *src, bigint size);
 
    /// Copy @a size entries from @a *this to @a dest.
    /** The given @a size should not exceed the Capacity() of @a *this and the
        destination, @a dest. */
-   inline void CopyTo(Memory &dest, int size) const;
+   inline void CopyTo(Memory &dest, bigint size) const;
 
    /// Copy @a size entries from @a *this to the host pointer @a dest.
    /** The given @a size should not exceed the Capacity() of @a *this. */
-   inline void CopyToHost(T *dest, int size) const;
+   inline void CopyToHost(T *dest, bigint size) const;
 
    /// Print the internal flags.
    /** This method can be useful for debugging. It is explicitly instantiated
@@ -567,7 +568,7 @@ public:
    /// If both the host and the device data are valid, compare their contents.
    /** This method can be useful for debugging. It is explicitly instantiated
        for Memory<T> with T = int and T = real_t. */
-   inline int CompareHostAndDevice(int size) const;
+   inline int CompareHostAndDevice(bigint size) const;
 
 private:
    // GCC 4.8 workaround: max_align_t is not in std.
@@ -956,7 +957,7 @@ inline void Memory<T>::Reset(MemoryType host_mt)
 }
 
 template <typename T>
-inline void Memory<T>::New(int size)
+inline void Memory<T>::New(bigint size)
 {
    capacity = size;
    flags = OWNS_HOST | VALID_HOST;
@@ -966,7 +967,7 @@ inline void Memory<T>::New(int size)
 }
 
 template <typename T>
-inline void Memory<T>::New(int size, MemoryType mt)
+inline void Memory<T>::New(bigint size, MemoryType mt)
 {
    capacity = size;
    const size_t bytes = size*sizeof(T);
@@ -978,7 +979,8 @@ inline void Memory<T>::New(int size, MemoryType mt)
 }
 
 template <typename T>
-inline void Memory<T>::New(int size, MemoryType host_mt, MemoryType device_mt)
+inline void Memory<T>::New(bigint size, MemoryType host_mt,
+                           MemoryType device_mt)
 {
    capacity = size;
    const size_t bytes = size*sizeof(T);
@@ -989,7 +991,7 @@ inline void Memory<T>::New(int size, MemoryType host_mt, MemoryType device_mt)
 }
 
 template <typename T>
-inline void Memory<T>::Wrap(T *ptr, int size, bool own)
+inline void Memory<T>::Wrap(T *ptr, bigint size, bool own)
 {
    h_ptr = ptr;
    capacity = size;
@@ -1011,7 +1013,7 @@ inline void Memory<T>::Wrap(T *ptr, int size, bool own)
 }
 
 template <typename T>
-inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
+inline void Memory<T>::Wrap(T *ptr, bigint size, MemoryType mt, bool own)
 {
    capacity = size;
    if (IsHostMemory(mt))
@@ -1036,7 +1038,7 @@ inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
 }
 
 template <typename T>
-inline void Memory<T>::Wrap(T *h_ptr_, T *d_ptr, int size, MemoryType h_mt_,
+inline void Memory<T>::Wrap(T *h_ptr_, T *d_ptr, bigint size, MemoryType h_mt_,
                             bool own, bool valid_host, bool valid_device)
 {
    h_mt = h_mt_;
@@ -1053,7 +1055,7 @@ inline void Memory<T>::Wrap(T *h_ptr_, T *d_ptr, int size, MemoryType h_mt_,
 }
 
 template <typename T>
-inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
+inline void Memory<T>::MakeAlias(const Memory &base, bigint offset, bigint size)
 {
    MFEM_ASSERT(0 <= offset, "invalid offset = " << offset);
    MFEM_ASSERT(0 <= size, "invalid size = " << size);
@@ -1140,7 +1142,7 @@ inline void Memory<T>::DeleteDevice(bool copy_to_host)
 }
 
 template <typename T>
-inline T &Memory<T>::operator[](int idx)
+inline T &Memory<T>::operator[](bigint idx)
 {
    MFEM_ASSERT((flags & VALID_HOST) && !(flags & VALID_DEVICE),
                "invalid host pointer access");
@@ -1148,7 +1150,7 @@ inline T &Memory<T>::operator[](int idx)
 }
 
 template <typename T>
-inline const T &Memory<T>::operator[](int idx) const
+inline const T &Memory<T>::operator[](bigint idx) const
 {
    MFEM_ASSERT((flags & VALID_HOST), "invalid host pointer access");
    return h_ptr[idx];
@@ -1189,7 +1191,7 @@ inline Memory<T>::operator const U*() const
 }
 
 template <typename T>
-inline T *Memory<T>::ReadWrite(MemoryClass mc, int size)
+inline T *Memory<T>::ReadWrite(MemoryClass mc, bigint size)
 {
    const size_t bytes = size * sizeof(T);
    if (!(flags & Registered))
@@ -1202,7 +1204,7 @@ inline T *Memory<T>::ReadWrite(MemoryClass mc, int size)
 }
 
 template <typename T>
-inline const T *Memory<T>::Read(MemoryClass mc, int size) const
+inline const T *Memory<T>::Read(MemoryClass mc, bigint size) const
 {
    const size_t bytes = size * sizeof(T);
    if (!(flags & Registered))
@@ -1215,7 +1217,7 @@ inline const T *Memory<T>::Read(MemoryClass mc, int size) const
 }
 
 template <typename T>
-inline T *Memory<T>::Write(MemoryClass mc, int size)
+inline T *Memory<T>::Write(MemoryClass mc, bigint size)
 {
    const size_t bytes = size * sizeof(T);
    if (!(flags & Registered))
@@ -1242,7 +1244,7 @@ inline void Memory<T>::Sync(const Memory &other) const
 }
 
 template <typename T>
-inline void Memory<T>::SyncAlias(const Memory &base, int alias_size) const
+inline void Memory<T>::SyncAlias(const Memory &base, bigint alias_size) const
 {
    // Assuming that if *this is registered then base is also registered.
    MFEM_ASSERT(!(flags & Registered) || (base.flags & Registered),
@@ -1279,7 +1281,7 @@ inline bool Memory<T>::DeviceIsValid() const
 }
 
 template <typename T>
-inline void Memory<T>::CopyFrom(const Memory &src, int size)
+inline void Memory<T>::CopyFrom(const Memory &src, bigint size)
 {
    MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
    if (size <= 0) { return; }
@@ -1300,7 +1302,7 @@ inline void Memory<T>::CopyFrom(const Memory &src, int size)
 }
 
 template <typename T>
-inline void Memory<T>::CopyFromHost(const T *src, int size)
+inline void Memory<T>::CopyFromHost(const T *src, bigint size)
 {
    MFEM_VERIFY(capacity>=size, "Incorrect size");
    if (size <= 0) { return; }
@@ -1321,13 +1323,13 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
 }
 
 template <typename T>
-inline void Memory<T>::CopyTo(Memory &dest, int size) const
+inline void Memory<T>::CopyTo(Memory &dest, bigint size) const
 {
    dest.CopyFrom(*this, size);
 }
 
 template <typename T>
-inline void Memory<T>::CopyToHost(T *dest, int size) const
+inline void Memory<T>::CopyToHost(T *dest, bigint size) const
 {
    MFEM_VERIFY(capacity>=size, "Incorrect size");
    if (size <= 0) { return; }
@@ -1359,7 +1361,7 @@ inline void Memory<T>::PrintFlags() const
 }
 
 template <typename T>
-inline int Memory<T>::CompareHostAndDevice(int size) const
+inline int Memory<T>::CompareHostAndDevice(bigint size) const
 {
    if (!(flags & VALID_HOST) || !(flags & VALID_DEVICE)) { return 0; }
    return MemoryManager::CompareHostAndDevice_(h_ptr, size*sizeof(T), flags);

general/stable3d.cpp

@@ -12,6 +12,7 @@
 
 #include "error.hpp"
 #include "stable3d.hpp"
+#include <limits>
 
 using namespace std;
 
@@ -90,6 +91,8 @@ int STable3D::Push (int r, int c, int f)
    node->Prev   = Rows[r];
    Rows[r] = node;
 
+   MFEM_VERIFY(NElem != std::numeric_limits<int>::max(),
+               "integer overflow error");
    NElem++;
    return (NElem-1);
 }

general/table.cpp

@@ -18,6 +18,7 @@
 #include "../general/mem_manager.hpp"
 #include <iostream>
 #include <iomanip>
+#include <limits>
 
 namespace mfem
 {
@@ -27,6 +28,8 @@ using namespace std;
 Table::Table(const Table &table1,
              const Table &table2, int offset)
 {
+   MFEM_VERIFY(!table1.UsingBigI() && !table2.UsingBigI(), "");
+
    MFEM_ASSERT(table1.size == table2.size,
                "Tables have different sizes can not merge.");
    size = table1.size;
@@ -60,6 +63,9 @@ Table::Table(const Table &table1,
              const Table &table2, int offset2,
              const Table &table3, int offset3)
 {
+   MFEM_VERIFY(!table1.UsingBigI() && !table2.UsingBigI() &&
+               !table3.UsingBigI(), "");
+
    MFEM_ASSERT(table1.size == table2.size,
                "Tables have different sizes can not merge.");
    MFEM_ASSERT(table1.size == table3.size,
@@ -98,17 +104,30 @@ Table::Table(const Table &table1,
 
 Table::Table (int dim, int connections_per_row)
 {
-   int i, j, sum = dim * connections_per_row;
+   bigint sum = bigint(dim) * connections_per_row;
 
    size = dim;
-   I.SetSize(size+1);
-   J.SetSize(sum);
-
-   I[0] = 0;
-   for (i = 1; i <= size; i++)
+   if (int(sum) == sum)
    {
-      I[i] = I[i-1] + connections_per_row;
-      for (j = I[i-1]; j < I[i]; j++) { J[j] = -1; }
+      I.SetSize(size+1);
+      J.SetSize(sum);
+      I[0] = 0;
+      for (int i = 1; i <= size; i++)
+      {
+         I[i] = I[i-1] + connections_per_row;
+         for (int j = I[i-1]; j < I[i]; j++) { J[j] = -1; }
+      }
+   }
+   else
+   {
+      bigI.SetSize(size+1);
+      J.SetSize(sum);
+      bigI[0] = 0;
+      for (int i = 1; i <= size; i++)
+      {
+         bigI[i] = bigI[i-1] + connections_per_row;
+         for (bigint j = bigI[i-1]; j < bigI[i]; j++) { J[j] = -1; }
+      }
    }
 }
 
@@ -139,77 +158,153 @@ void Table::MakeI(int nrows)
 
 void Table::MakeJ()
 {
-   int i, j, k;
+   bigint nnz;
 
-   for (k = i = 0; i < size; i++)
+   if (!UsingBigI())
    {
-      j = I[i], I[i] = k, k += j;
+      nnz = 0;
+      int nnz_int = 0;
+      for (int i = 0; i < size; i++)
+      {
+         const int row_size = I[i];
+         I[i] = nnz_int;
+         nnz_int += row_size;
+         nnz += row_size;
+         if (nnz_int != nnz) // check for overflow
+         {
+            bigI.SetSize(size+1);
+            for (int j = 0; j <= i; j++) { bigI[j] = I[j]; }
+            for (i++ ; i < size; i++)
+            {
+               bigI[i] = nnz;
+               nnz += I[i];
+            }
+            bigI[size] = nnz;
+            I.DeleteAll();
+            goto I_is_updated;
+         }
+      }
+      I[size] = nnz_int;
+      nnz = nnz_int;
+   I_is_updated: ;
+   }
+   else
+   {
+      nnz = 0;
+      for (int i = 0; i < size; i++)
+      {
+         const bigint row_size = bigI[i];
+         bigI[i] = nnz;
+         nnz += row_size;
+      }
+      bigI[size] = nnz;
    }
 
-   J.SetSize(I[size]=k);
+   J.SetSize(nnz);
 }
 
 void Table::AddConnections(int r, const int *c, int nc)
 {
-   int *jp = J+I[r];
+   int *jp = GetRow(r);
 
    for (int i = 0; i < nc; i++)
    {
       jp[i] = c[i];
    }
-   I[r] += nc;
+   UsingBigI() ? bigI[r] += nc : I[r] += nc;
 }
 
 void Table::ShiftUpI()
 {
-   for (int i = size; i > 0; i--)
+   if (!UsingBigI())
    {
-      I[i] = I[i-1];
+      for (int i = size; i > 0; i--)
+      {
+         I[i] = I[i-1];
+      }
+      I[0] = 0;
+   }
+   else
+   {
+      for (int i = size; i > 0; i--)
+      {
+         bigI[i] = bigI[i-1];
+      }
+      bigI[0] = 0;
    }
-   I[0] = 0;
 }
 
 void Table::SetSize(int dim, int connections_per_row)
 {
-   SetDims (dim, dim * connections_per_row);
+   SetDims(dim, bigint(dim) * connections_per_row);
 
    if (size > 0)
    {
-      I[0] = 0;
-      for (int i = 0, j = 0; i < size; i++)
+      if (!UsingBigI())
       {
-         int end = I[i] + connections_per_row;
-         I[i+1] = end;
-         for ( ; j < end; j++) { J[j] = -1; }
+         I[0] = 0;
+         for (int i = 0, j = 0; i < size; i++)
+         {
+            int end = I[i] + connections_per_row;
+            I[i+1] = end;
+            for ( ; j < end; j++) { J[j] = -1; }
+         }
+      }
+      else
+      {
+         bigint j = 0;
+         bigI[0] = 0;
+         for (int i = 0; i < size; i++)
+         {
+            bigint end = bigI[i] + connections_per_row;
+            bigI[i+1] = end;
+            for ( ; j < end; j++) { J[j] = -1; }
+         }
       }
    }
 }
 
-void Table::SetDims(int rows, int nnz)
+void Table::SetDims(int rows, bigint nnz)
 {
-   int j;
-
-   j = (I) ? (I[size]) : (0);
-   if (size != rows)
+   const bool new_use_big_i = (bigint(int(nnz)) != nnz);
+   if (size != rows || new_use_big_i != UsingBigI())
    {
       size = rows;
-      (rows >= 0) ? I.SetSize(rows+1) : I.DeleteAll();
+      if (new_use_big_i != UsingBigI())
+      {
+         UsingBigI() ? bigI.DeleteAll() : I.DeleteAll();
+      }
+      if (size >= 0)
+      {
+         new_use_big_i ? bigI.SetSize(size+1) : I.SetSize(size+1);
+      }
+      else
+      {
+         new_use_big_i ? bigI.DeleteAll() : I.DeleteAll();
+      }
    }
 
-   if (j != nnz)
-   {
-      (nnz > 0) ? J.SetSize(nnz) : J.DeleteAll();
-   }
+   (nnz > 0) ? J.SetSize(nnz) : J.DeleteAll();
 
    if (size >= 0)
    {
-      I[0] = 0;
-      I[size] = nnz;
+      if (!UsingBigI())
+      {
+         I[0] = 0;
+         I[size] = int(nnz);
+      }
+      else
+      {
+         bigI[0] = 0;
+         bigI[size] = nnz;
+      }
    }
 }
 
 int Table::operator()(int i, int j) const
 {
+   MFEM_VERIFY(!UsingBigI(), "");
+
    if ( i>=size || i<0 )
    {
       return -1;
@@ -236,7 +331,8 @@ void Table::GetRow(int i, Array<int> &row) const
                << size << ')');
 
    HostReadJ();
-   HostReadI();
+   if (UsingBigI()) { HostReadBigI(); }
+   else { HostReadI(); }
 
    row.SetSize(RowSize(i));
    row.Assign(GetRow(i));
@@ -244,14 +340,25 @@ void Table::GetRow(int i, Array<int> &row) const
 
 void Table::SortRows()
 {
-   for (int r = 0; r < size; r++)
+   if (!UsingBigI())
    {
-      std::sort(J + I[r], J + I[r+1]);
+      for (int r = 0; r < size; r++)
+      {
+         std::sort(J.GetData()+I[r], J.GetData()+I[r+1]);
+      }
+   }
+   else
+   {
+      for (int r = 0; r < size; r++)
+      {
+         std::sort(J.GetData()+bigI[r], J.GetData()+bigI[r+1]);
+      }
    }
 }
 
 void Table::SetIJ(int *newI, int *newJ, int newsize)
 {
+   if (UsingBigI()) { bigI.DeleteAll(); }
    if (newsize >= 0)
    {
       size = newsize;
@@ -262,6 +369,8 @@ void Table::SetIJ(int *newI, int *newJ, int newsize)
 
 int Table::Push(int i, int j)
 {
+   MFEM_VERIFY(!UsingBigI(), "");
+
    MFEM_ASSERT(i >=0 &&
                i<size, "Index out of bounds.  i = " << i << " size " << size);
 
@@ -286,6 +395,8 @@ int Table::Push(int i, int j)
 
 void Table::Finalize()
 {
+   MFEM_VERIFY(!UsingBigI(), "");
+
    int i, j, end, sum = 0, n = 0, newI = 0;
 
    for (i=0; i<I[size]; i++)
@@ -324,14 +435,16 @@ void Table::MakeFromList(int nrows, const Array<Connection> &list)
    Clear();
 
    size = nrows;
-   int nnz = list.Size();
+   const bigint nnz = list.Size();
+   const bool use_big_i = (bigint(int(nnz)) != nnz);
 
-   I.SetSize(size+1);
+   use_big_i ? bigI.SetSize(size+1) : I.SetSize(size+1);
    J.SetSize(nnz);
 
-   for (int i = 0, k = 0; i <= size; i++)
+   bigint k = 0;
+   for (int i = 0; i <= size; i++)
    {
-      I[i] = k;
+      use_big_i ? bigI[i] = k : I[i] = int(k);
       while (k < nnz && list[k].from == i)
       {
          J[k] = list[k].to;
@@ -342,30 +455,25 @@ void Table::MakeFromList(int nrows, const Array<Connection> &list)
 
 int Table::Width() const
 {
-   int width = -1, nnz = (size >= 0) ? I[size] : 0;
-   for (int k = 0; k < nnz; k++)
-   {
-      if (J[k] > width) { width = J[k]; }
-   }
-   return width + 1;
+   return (J.Size() > 0) ? J.Max() + 1 : 0;
 }
 
 void Table::Print(std::ostream & os, int width) const
 {
-   int i, j;
-
-   for (i = 0; i < size; i++)
+   for (int i = 0; i < size; i++)
    {
       os << "[row " << i << "]\n";
-      for (j = I[i]; j < I[i+1]; j++)
+      const int row_size = RowSize(i);
+      const int *row = GetRow(i);
+      for (int j = 0; j < row_size; j++)
       {
-         os << setw(5) << J[j];
-         if ( !((j+1-I[i]) % width) )
+         os << setw(5) << row[j];
+         if ( !((j+1) % width) )
          {
             os << '\n';
          }
       }
-      if ((j-I[i]) % width)
+      if (row_size % width)
       {
          os << '\n';
       }
@@ -378,9 +486,11 @@ void Table::PrintMatlab(std::ostream & os) const
 
    for (i = 0; i < size; i++)
    {
-      for (j = I[i]; j < I[i+1]; j++)
+      const int row_size = RowSize(i);
+      const int *row = GetRow(i);
+      for (j = 0; j < row_size; j++)
       {
-         os << i << " " << J[j] << " 1. \n";
+         os << i << " " << row[j] << " 1. \n";
       }
    }
 
@@ -391,35 +501,51 @@ void Table::Save(std::ostream &os) const
 {
    os << size << '\n';
 
-   for (int i = 0; i <= size; i++)
+   if (!UsingBigI())
    {
-      os << I[i] << '\n';
+      I.Save(os, 1);
    }
-   for (int i = 0, nnz = I[size]; i < nnz; i++)
+   else
    {
-      os << J[i] << '\n';
+      bigI.Save(os, 1);
    }
+   J.Save(os, 1);
 }
 
 void Table::Load(std::istream &in)
 {
+   Clear();
+
    in >> size;
    I.SetSize(size+1);
    for (int i = 0; i <= size; i++)
    {
-      in >> I[i];
-   }
-   int nnz = I[size];
-   J.SetSize(nnz);
-   for (int j = 0; j < nnz; j++)
-   {
-      in >> J[j];
+      bigint big_offset;
+      in >> big_offset;
+      const int offset = int(big_offset);
+      if (bigint(offset) != big_offset)
+      {
+         // switch to using bigI instead of I
+         bigI.SetSize(size+1);
+         for (int j = 0; j < i; j++) { bigI[j] = I[j]; }
+         I.DeleteAll();
+         bigI[i] = big_offset;
+         for (i++; i <= size; i++)
+         {
+            in >> bigI[i];
+         }
+         break;
+      }
+      I[i] = offset;
    }
+   J.SetSize(UsingBigI() ? bigI[size] : I[size]);
+   J.Load(in, 1);
 }
 
 void Table::Clear()
 {
    I.DeleteAll();
+   bigI.DeleteAll();
    J.DeleteAll();
    size = -1;
 }
@@ -436,50 +562,72 @@ void Table::Swap(Table & other)
 
 std::size_t Table::MemoryUsage() const
 {
-   if (size < 0 || I == NULL) { return 0; }
-   return (size+1 + I[size]) * sizeof(int);
+   return I.MemoryUsage() + bigI.MemoryUsage() + J.MemoryUsage();
 }
 
-void Transpose(const Table &A, Table &At, int ncols_A_)
+template <typename TI, typename TJ>
+void TransposeImpl(const TI *i_A, const TJ *j_A, const TJ nrows_A,
+                   const TJ ncols_A, const TI nnz_A, TI *i_At, TJ *j_At)
 {
-   const int *i_A     = A.GetI();
-   const int *j_A     = A.GetJ();
-   const int  nrows_A = A.Size();
-   const int  ncols_A = (ncols_A_ < 0) ? A.Width() : ncols_A_;
-   const int  nnz_A   = i_A[nrows_A];
-
-   At.SetDims (ncols_A, nnz_A);
-
-   int *i_At = At.GetI();
-   int *j_At = At.GetJ();
-
-   for (int i = 0; i <= ncols_A; i++)
+   for (TJ i = 0; i <= ncols_A; i++)
    {
       i_At[i] = 0;
    }
-   for (int i = 0; i < nnz_A; i++)
+   for (TI i = 0; i < nnz_A; i++)
    {
       i_At[j_A[i]+1]++;
    }
-   for (int i = 1; i < ncols_A; i++)
+   for (TJ i = 1; i < ncols_A; i++)
    {
       i_At[i+1] += i_At[i];
    }
 
-   for (int i = 0; i < nrows_A; i++)
+   for (TJ i = 0; i < nrows_A; i++)
    {
-      for (int j = i_A[i]; j < i_A[i+1]; j++)
+      for (TI j = i_A[i]; j < i_A[i+1]; j++)
       {
          j_At[i_At[j_A[j]]++] = i;
       }
    }
-   for (int i = ncols_A; i > 0; i--)
+   for (TJ i = ncols_A; i > 0; i--)
    {
       i_At[i] = i_At[i-1];
    }
    i_At[0] = 0;
 }
 
+void Transpose(const Table &A, Table &At, int ncols_A_)
+{
+   const int *j_A     = A.HostReadJ();
+   const int  nrows_A = A.Size();
+   const int  ncols_A = (ncols_A_ < 0) ? A.Width() : ncols_A_;
+
+   if (!A.UsingBigI())
+   {
+      const int *i_A   = A.HostReadI();
+      const int  nnz_A = i_A[nrows_A];
+
+      At.SetDims(ncols_A, nnz_A);
+
+      int *i_At = At.HostWriteI();
+      int *j_At = At.HostWriteJ();
+
+      TransposeImpl(i_A, j_A, nrows_A, ncols_A, nnz_A, i_At, j_At);
+   }
+   else
+   {
+      const bigint *i_A   = A.HostReadBigI();
+      const bigint  nnz_A = i_A[nrows_A];
+
+      At.SetDims(ncols_A, nnz_A);
+
+      bigint *i_At = At.HostWriteBigI();
+      int    *j_At = At.HostWriteJ();
+
+      TransposeImpl(i_A, j_A, nrows_A, ncols_A, nnz_A, i_At, j_At);
+   }
+}
+
 Table * Transpose(const Table &A)
 {
    Table * At = new Table;
@@ -504,6 +652,8 @@ void Transpose(const Array<int> &A, Table &At, int ncols_A_)
 
 void Mult(const Table &A, const Table &B, Table &C)
 {
+   MFEM_VERIFY(!A.UsingBigI() && !B.UsingBigI(), "");
+
    int  i, j, k, l, m;
    const int *i_A     = A.GetI();
    const int *j_A     = A.GetJ();
@@ -641,6 +791,8 @@ int DSTable::Push_(int r, int c)
    n->Index  = NumEntries;
    n->Prev   = Rows[r];
    Rows[r]   = n;
+   MFEM_VERIFY(NumEntries != std::numeric_limits<int>::max(),
+               "integer overflow error");
    return (NumEntries++);
 }
 

general/table.hpp

@@ -42,6 +42,7 @@ struct Connection
 class Table
 {
 protected:
+   // FIXME: this member can mess up the default move ctor ?!!
    int size; ///< The number of TYPE I elements.
 
    /// @name Arrays for the connectivity information in the CSR storage.
@@ -50,6 +51,10 @@ protected:
    /// The length of the I array is 'size + 1',
    Array<int> I;
 
+   /** @brief Alternative to the I array. Used when the number of connections
+       overflows the int type. */
+   Array<bigint> bigI;
+
    /// @brief The length of the J array is equal to the number of connections
    /// between TYPE I and TYPE II elements.
    Array<int> J;
@@ -83,21 +88,25 @@ public:
    /// @name Used together with the default constructor
    /// @{
    void MakeI(int nrows);
-   void AddAColumnInRow(int r) { I[r]++; }
-   void AddColumnsInRow(int r, int ncol) { I[r] += ncol; }
+   void AddAColumnInRow(int r) { UsingBigI() ? bigI[r]++ : I[r]++; }
+   void AddColumnsInRow (int r, int ncol)
+   { UsingBigI() ? bigI[r] += ncol : I[r] += ncol; }
    void MakeJ();
-   void AddConnection(int r, int c) { J[I[r]++] = c; }
+   void AddConnection (int r, int c)
+   { UsingBigI() ? J[bigI[r]++] = c : J[I[r]++] = c; }
    void AddConnections(int r, const int *c, int nc);
    void ShiftUpI();
    /// @}
 
+   bool UsingBigI() const { return !bigI.IsEmpty(); }
+
    /// Set the size and the number of connections for the table.
    void SetSize(int dim, int connections_per_row);
 
    /// @brief Set the rows and the number of all connections for the table.
    ///
    /// Does NOT initialize the whole array I ! (I[0]=0 and I[rows]=nnz only)
-   void SetDims(int rows, int nnz);
+   void SetDims(int rows, bigint nnz);
 
    /// Returns the number of TYPE I elements.
    inline int Size() const { return size; }
@@ -107,7 +116,7 @@ public:
    /// If Finalize() is not called, it returns the number of possible
    /// connections established by the used constructor. Otherwise, it is exactly
    /// the number of established connections after calling Finalize(). */
-   inline int Size_of_connections() const { return J.Size(); }
+   inline bigint Size_of_connections() const { return J.Size(); }
 
    /// @brief Returns index of the connection between element i of TYPE I and
    /// element j of TYPE II.
@@ -119,27 +128,94 @@ public:
    /// Return row i in array row (the Table must be finalized)
    void GetRow(int i, Array<int> &row) const;
 
-   int RowSize(int i) const { return I[i+1] - I[i]; }
+   int RowSize(int i) const
+   { return UsingBigI() ? int(bigI[i+1]-bigI[i]): I[i+1]-I[i]; }
 
-   const int *GetRow(int i) const { return J.GetMemory() + I[i]; }
-   int *GetRow(int i) { return J.GetMemory() + I[i]; }
+   const int *GetRow(int i) const
+   { return UsingBigI() ? J.GetData()+bigI[i] : J.GetData()+I[i]; }
+
+   int *GetRow(int i)
+   { return UsingBigI() ? J.GetData()+bigI[i] : J.GetData()+I[i]; }
+
+   int *GetI()
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.GetData();
+   }
 
-   int *GetI() { return I.GetData(); }
    int *GetJ() { return J.GetData(); }
-   const int *GetI() const { return I.GetData(); }
+
+   const int *GetI() const
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.GetData();
+   }
+
    const int *GetJ() const { return J.GetData(); }
 
-   Memory<int> &GetIMemory() { return I.GetMemory(); }
+   Memory<int> &GetIMemory()
+   { MFEM_ASSERT(!UsingBigI(), ""); return I.GetMemory(); }
+
    Memory<int> &GetJMemory() { return J.GetMemory(); }
-   const Memory<int> &GetIMemory() const { return I.GetMemory(); }
+
+   const Memory<int> &GetIMemory() const
+   { MFEM_ASSERT(!UsingBigI(), ""); return I.GetMemory(); }
+
    const Memory<int> &GetJMemory() const { return J.GetMemory(); }
 
-   const int *ReadI(bool on_dev = true) const { return I.Read(on_dev); }
-   int *WriteI(bool on_dev = true) { return I.Write(on_dev); }
-   int *ReadWriteI(bool on_dev = true) { return I.ReadWrite(on_dev); }
-   const int *HostReadI() const { return I.HostRead(); }
-   int *HostWriteI() { return I.HostWrite(); }
-   int *HostReadWriteI() { return I.HostReadWrite(); }
+   const int *ReadI(bool on_dev = true) const
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.Read(on_dev);
+   }
+
+   int *WriteI(bool on_dev = true)
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.Write(on_dev);
+   }
+
+   int *ReadWriteI(bool on_dev = true)
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.ReadWrite(on_dev);
+   }
+
+   const int *HostReadI() const
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.HostRead();
+   }
+
+   int *HostWriteI()
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.HostWrite();
+   }
+
+   int *HostReadWriteI()
+   {
+      MFEM_ASSERT(!UsingBigI(), "");
+      return I.HostReadWrite();
+   }
+
+   const bigint *HostReadBigI() const
+   {
+      MFEM_ASSERT(UsingBigI(), "");
+      return bigI.HostRead();
+   }
+
+   bigint *HostWriteBigI()
+   {
+      MFEM_ASSERT(UsingBigI(), "");
+      return bigI.HostWrite();
+   }
+
+   bigint *HostReadWriteBigI()
+   {
+      MFEM_ASSERT(UsingBigI(), "");
+      return bigI.HostReadWrite();
+   }
 
    const int *ReadJ(bool on_dev = true) const { return J.Read(on_dev); }
    int *WriteJ(bool on_dev = true) { return J.Write(on_dev); }
@@ -181,7 +257,7 @@ public:
    int Width() const;
 
    /// Releases ownership of and null-ifies the data.
-   void LoseData() { size = -1; I.LoseData(); J.LoseData(); }
+   void LoseData() { size = -1; I.LoseData(); bigI.LoseData(); J.LoseData(); }
 
    /// Prints the table to the stream @a out.
    void Print(std::ostream & out = mfem::out, int width = 4) const;

general/text.hpp

@@ -50,48 +50,6 @@ inline void filter_dos(std::string &line)
    }
 }
 
-/** @brief Read a string formatted using std::quoted. Return nonzero on error.
-
-    The stream @a in must begin with @a delim. After clearing @a result and
-    extracting the opening @a delim, characters are extracted from @a in and
-    processed as follows:
-    - if the character is @a delim, return 0;
-    - if the character is different from @a escape, it is appended to @a result;
-    - if the character is @a escape, the next character from @a in is extracted
-      and if it is one of @a delim or @a escape, it is appended to @a result;
-      otherwise, both @a escape and the character after it are appended to
-      @a result; note that the latter case is not possible if the input was
-      formatted with std::quoted with the same @a delim and @a escape
-      characters.
-
-    If the stream @a in does not begin with @a delim, error code 1 is returned.
-    If reading the stream fails, error code 2 is returned. On success, zero is
-    returned and the closing @a delim character is the last character extracted
-    from @a in. */
-inline int parse_quoted_string(std::string &result, std::istream &in,
-                               char delim = '"', char escape = '\\')
-{
-   using tt = std::string::traits_type;  // std::char_traits<char>
-   auto equal = [](tt::int_type c1, tt::char_type c2) -> bool
-   {
-      return tt::eq_int_type(c1, tt::to_int_type(c2));
-   };
-   result.clear();
-   if (!equal(in.peek(), delim)) { return 1; }
-   in.get();  // extract delim
-   for (auto c = in.get(); !equal(c, delim); c = in.get())
-   {
-      if (equal(c, escape))
-      {
-         c = in.get();
-         if (!equal(c, escape) && !equal(c, delim)) { result += escape; }
-      }
-      if (!in) { return 2; }
-      result += tt::to_char_type(c);
-   }
-   return 0;
-}
-
 /// Convert an integer to a 0-padded string with the given number of @a digits
 inline std::string to_padded_string(int i, int digits)
 {

linalg/dtensor.hpp

@@ -24,13 +24,14 @@ class TensorInd
 {
 public:
    MFEM_HOST_DEVICE
-   static inline int result(const int* sizes, T first, Args... args)
+   static inline bigint result(const int* sizes, T first, Args... args)
    {
 #if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
       MFEM_ASSERT(first<sizes[N-1],"Trying to access out of boundary.");
 #endif
-      return static_cast<int>(first + sizes[N - 1] * TensorInd < N + 1, Dim, Args... >
-                              ::result(sizes, args...));
+      return static_cast<bigint>(
+                first + sizes[N - 1] * TensorInd < N + 1, Dim, Args... >
+                ::result(sizes, args...));
    }
 };
 
@@ -40,13 +41,13 @@ class TensorInd<Dim, Dim, T, Args...>
 {
 public:
    MFEM_HOST_DEVICE
-   static inline int result(const int* sizes, T first, Args... args)
+   static inline bigint result(const int* sizes, T first, Args... args)
    {
 #if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
       MFEM_ASSERT(first<static_cast<T>(sizes[Dim-1]),
                   "Trying to access out of boundary.");
 #endif
-      return static_cast<int>(first);
+      return static_cast<bigint>(first);
    }
 };
 
@@ -57,7 +58,7 @@ class Init
 {
 public:
    MFEM_HOST_DEVICE
-   static inline int result(int* sizes, T first, Args... args)
+   static inline bigint result(int* sizes, T first, Args... args)
    {
       sizes[N - 1] = first;
       return first * Init < N + 1, Dim, Args... >::result(sizes, args...);
@@ -70,10 +71,10 @@ class Init<Dim, Dim, T, Args...>
 {
 public:
    MFEM_HOST_DEVICE
-   static inline int result(int* sizes, T first, Args... args)
+   static inline bigint result(int* sizes, T first, Args... args)
    {
       sizes[Dim - 1] = first;
-      return first;
+      return static_cast<bigint>(first);
    }
 };
 
@@ -83,7 +84,7 @@ template<int Dim, typename Scalar = real_t>
 class DeviceTensor
 {
 protected:
-   int capacity;
+   bigint capacity;
    Scalar *data;
    int sizes[Dim];
 
@@ -99,8 +100,7 @@ public:
    {
       static_assert(sizeof...(args) == Dim, "Wrong number of arguments");
       // Initialize sizes, and compute the number of values
-      const long int nb = Init<1, Dim, Args...>::result(sizes, args...);
-      capacity = nb;
+      capacity = Init<1, Dim, Args...>::result(sizes, args...);
       data = (capacity > 0) ? data_ : nullptr;
    }
 
@@ -122,7 +122,7 @@ public:
    }
 
    /// Subscript operator where the tensor is viewed as a 1D array.
-   MFEM_HOST_DEVICE inline Scalar& operator[](int i) const
+   MFEM_HOST_DEVICE inline Scalar& operator[](bigint i) const
    {
       return data[i];
    }

linalg/hypre.cpp

@@ -317,9 +317,6 @@ void HypreParVector::WrapHypreParVector(hypre_ParVector *y, bool owner)
 
 Vector * HypreParVector::GlobalVector() const
 {
-   MFEM_VERIFY(size > 0,
-               "GlobalVector method can only be called on vectors wherein each "
-               "process owns one or more entries");
    hypre_Vector *hv = hypre_ParVectorToVectorAll(*this);
    Vector *v = new Vector(hv->data, internal::to_int(hv->size));
    v->MakeDataOwner();

linalg/particlevector.cpp

@@ -16,7 +16,8 @@ namespace mfem
 
 void ParticleVector::GrowSize(int min_num_vectors, bool keep_data)
 {
-   const int nsize = std::max(min_num_vectors*vdim, 2 * data.Capacity());
+   const bigint nsize = std::max(bigint(min_num_vectors)*vdim,
+                                 2 * data.Capacity());
    Memory<real_t> p(nsize, data.GetMemoryType());
    if (keep_data) { p.CopyFrom(data, size); }
    p.UseDevice(data.UseDevice());

linalg/petsc.cpp

@@ -38,13 +38,6 @@
 #if PETSC_VERSION_LT(3,19,0)
 #define PETSC_SUCCESS 0
 #endif
-#if PETSC_VERSION_LT(3,23,0)
-#define PetscContainerSetCtxDestroy(A,B) PetscContainerSetUserDestroy(A,B)
-typedef PetscErrorCode (PetscCtxDestroyFn)(void**);
-#endif
-#if PETSC_VERSION_LT(3,24,0)
-typedef PetscErrorCode KSPMonitorFn(KSP,PetscInt,PetscReal,void*);
-#endif
 
 #include <fstream>
 #include <iomanip>
@@ -84,17 +77,13 @@ static PetscErrorCode __mfem_mat_shell_apply_transpose(Mat,Vec,Vec);
 static PetscErrorCode __mfem_mat_shell_destroy(Mat);
 static PetscErrorCode __mfem_mat_shell_copy(Mat,Mat,MatStructure);
 #if PETSC_VERSION_LT(3,23,0)
-typedef void *PetscCtxRt;
-#elif PETSC_VERSION_LT(3,25,0)
-typedef void **PetscCtxRt;
-#endif
-static PetscErrorCode __mfem_array_container_destroy(PetscCtxRt);
-static PetscErrorCode __mfem_matarray_container_destroy(PetscCtxRt);
-#if PETSC_VERSION_LT(3,23,0)
-static PetscErrorCode __mfem_monitor_ctx_destroy(void**);
+static PetscErrorCode __mfem_array_container_destroy(void*);
+static PetscErrorCode __mfem_matarray_container_destroy(void *);
 #else
-static PetscErrorCode __mfem_monitor_ctx_destroy(PetscCtxRt);
+static PetscErrorCode __mfem_array_container_destroy(void**);
+static PetscErrorCode __mfem_matarray_container_destroy(void**);
 #endif
+static PetscErrorCode __mfem_monitor_ctx_destroy(void**);
 
 // auxiliary functions
 static PetscErrorCode Convert_Array_IS(MPI_Comm,bool,const mfem::Array<int>*,
@@ -1328,7 +1317,11 @@ BlockDiagonalConstructor(MPI_Comm comm,
 
          ierr = PetscContainerCreate(comm,&c); CCHKERRQ(comm,ierr);
          ierr = PetscContainerSetPointer(c,ptrs[i]); CCHKERRQ(comm,ierr);
+#if PETSC_VERSION_LT(3,23,0)
+         ierr = PetscContainerSetUserDestroy(c,__mfem_array_container_destroy);
+#else
          ierr = PetscContainerSetCtxDestroy(c,__mfem_array_container_destroy);
+#endif
          CCHKERRQ(comm,ierr);
          ierr = PetscObjectCompose((PetscObject)A,names[i],(PetscObject)c);
          CCHKERRQ(comm,ierr);
@@ -1655,7 +1648,11 @@ void PetscParMatrix::ConvertOperator(MPI_Comm comm, const Operator &op, Mat* A,
          PetscContainer c;
          ierr = PetscContainerCreate(comm,&c); CCHKERRQ(comm,ierr);
          ierr = PetscContainerSetPointer(c,vmatsl2l); PCHKERRQ(c,ierr);
+#if PETSC_VERSION_LT(3,23,0)
+         ierr = PetscContainerSetUserDestroy(c,__mfem_matarray_container_destroy);
+#else
          ierr = PetscContainerSetCtxDestroy(c,__mfem_matarray_container_destroy);
+#endif
          PCHKERRQ(c,ierr);
          ierr = PetscObjectCompose((PetscObject)(*A),"_MatIS_PtAP_l2l",(PetscObject)c);
          PCHKERRQ((*A),ierr);
@@ -1751,7 +1748,11 @@ void PetscParMatrix::ConvertOperator(MPI_Comm comm, const Operator &op, Mat* A,
 
             ierr = PetscContainerCreate(PETSC_COMM_SELF,&c); PCHKERRQ(B,ierr);
             ierr = PetscContainerSetPointer(c,ptrs[i]); PCHKERRQ(B,ierr);
+#if PETSC_VERSION_LT(3,23,0)
+            ierr = PetscContainerSetUserDestroy(c,__mfem_array_container_destroy);
+#else
             ierr = PetscContainerSetCtxDestroy(c,__mfem_array_container_destroy);
+#endif
             PCHKERRQ(B,ierr);
             ierr = PetscObjectCompose((PetscObject)(B),names[i],(PetscObject)c);
             PCHKERRQ(B,ierr);
@@ -2197,7 +2198,11 @@ PetscParMatrix * RAP(PetscParMatrix *Rt, PetscParMatrix *A, PetscParMatrix *P)
          ierr = PetscContainerCreate(PetscObjectComm((PetscObject)B),&c);
          PCHKERRQ(B,ierr);
          ierr = PetscContainerSetPointer(c,vmatsl2l); PCHKERRQ(c,ierr);
+#if PETSC_VERSION_LT(3,23,0)
+         ierr = PetscContainerSetUserDestroy(c,__mfem_matarray_container_destroy);
+#else
          ierr = PetscContainerSetCtxDestroy(c,__mfem_matarray_container_destroy);
+#endif
          PCHKERRQ(c,ierr);
          ierr = PetscObjectCompose((PetscObject)B,"_MatIS_PtAP_l2l",(PetscObject)c);
          PCHKERRQ(B,ierr);
@@ -2480,6 +2485,7 @@ void PetscSolver::SetMaxIter(int max_iter)
 
 void PetscSolver::SetPrintLevel(int plev)
 {
+   typedef PetscErrorCode (*myPetscFunc)(void**);
    PetscViewerAndFormat *vf = NULL;
    PetscViewer viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm(obj));
 
@@ -2492,6 +2498,7 @@ void PetscSolver::SetPrintLevel(int plev)
    {
       // there are many other options, see the function KSPSetFromOptions() in
       // src/ksp/ksp/interface/itcl.c
+      typedef PetscErrorCode (*myMonitor)(KSP,PetscInt,PetscReal,void*);
       KSP ksp = (KSP)obj;
       if (plev >= 0)
       {
@@ -2500,29 +2507,29 @@ void PetscSolver::SetPrintLevel(int plev)
       if (plev == 1)
       {
 #if PETSC_VERSION_LT(3,15,0)
-         ierr = KSPMonitorSet(ksp,(KSPMonitorFn *)KSPMonitorDefault,vf,
+         ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorDefault,vf,
 #else
-         ierr = KSPMonitorSet(ksp,(KSPMonitorFn *)KSPMonitorResidual,vf,
+         ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorResidual,vf,
 #endif
-                              (PetscCtxDestroyFn *)PetscViewerAndFormatDestroy);
+                              (myPetscFunc)PetscViewerAndFormatDestroy);
          PCHKERRQ(ksp,ierr);
       }
       else if (plev > 1)
       {
          ierr = KSPSetComputeSingularValues(ksp,PETSC_TRUE); PCHKERRQ(ksp,ierr);
-         ierr = KSPMonitorSet(ksp,(KSPMonitorFn *)KSPMonitorSingularValue,vf,
-                              (PetscCtxDestroyFn *)PetscViewerAndFormatDestroy);
+         ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorSingularValue,vf,
+                              (myPetscFunc)PetscViewerAndFormatDestroy);
          PCHKERRQ(ksp,ierr);
          if (plev > 2)
          {
             ierr = PetscViewerAndFormatCreate(viewer,PETSC_VIEWER_DEFAULT,&vf);
             PCHKERRQ(viewer,ierr);
 #if PETSC_VERSION_LT(3,15,0)
-            ierr = KSPMonitorSet(ksp,(KSPMonitorFn *)KSPMonitorTrueResidualNorm,vf,
+            ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorTrueResidualNorm,vf,
 #else
-            ierr = KSPMonitorSet(ksp,(KSPMonitorFn *)KSPMonitorTrueResidual,vf,
+            ierr = KSPMonitorSet(ksp,(myMonitor)KSPMonitorTrueResidual,vf,
 #endif
-                                 (PetscCtxDestroyFn *)PetscViewerAndFormatDestroy);
+                                 (myPetscFunc)PetscViewerAndFormatDestroy);
             PCHKERRQ(ksp,ierr);
          }
       }
@@ -2538,7 +2545,7 @@ void PetscSolver::SetPrintLevel(int plev)
       if (plev > 0)
       {
          ierr = SNESMonitorSet(snes,(myMonitor)SNESMonitorDefault,vf,
-                               (PetscCtxDestroyFn *)PetscViewerAndFormatDestroy);
+                               (myPetscFunc)PetscViewerAndFormatDestroy);
          PCHKERRQ(snes,ierr);
       }
    }
@@ -4156,31 +4163,20 @@ void PetscNonlinearSolver::SetUpdate(void (*update)(Operator *,int,
 void PetscNonlinearSolver::Mult(const Vector &b, Vector &x) const
 {
    SNES snes = (SNES)obj;
-   MPI_Comm comm = PetscObjectComm(obj);
 
-   // Reduction needed: some processes may have null local size while others don't,
-   // and VecPlaceArray (used by PlaceMemory) is a logically collective operation.
-   PetscBool b_nonempty = b.Size() ? PETSC_TRUE : PETSC_FALSE;
-#if PETSC_VERSION_LT(3,24,0)
-   mpiierr = MPI_Allreduce(MPI_IN_PLACE,&b_nonempty,1,MPIU_BOOL,MPI_LOR,comm);
-#else
-   mpiierr = MPI_Allreduce(MPI_IN_PLACE,&b_nonempty,1,MPI_C_BOOL,MPI_LOR,comm);
-#endif
-   CCHKERRQ(comm,mpiierr);
-
-   // Always create B with allocate=false so that PlaceMemory can be called on
-   // it regardless of whether b was empty on a previous call.
-   if (!B) { B = new PetscParVector(comm, *this, true, false); }
-   if (!X) { X = new PetscParVector(comm, *this, false, false); }
+   bool b_nonempty = b.Size();
+   if (!B) { B = new PetscParVector(PetscObjectComm(obj), *this, true); }
+   if (!X) { X = new PetscParVector(PetscObjectComm(obj), *this, false, false); }
    X->PlaceMemory(x.GetMemory(),iterative_mode);
    if (b_nonempty) { B->PlaceMemory(b.GetMemory()); }
+   else { *B = 0.0; }
 
    Customize();
 
    if (!iterative_mode) { *X = 0.; }
 
-   // Solve the system. Pass nullptr for b when empty (PETSc treats it as zero RHS).
-   ierr = SNESSolve(snes, b_nonempty ? B->x : nullptr, X->x); PCHKERRQ(snes, ierr);
+   // Solve the system.
+   ierr = SNESSolve(snes, B->x, X->x); PCHKERRQ(snes, ierr);
    X->ResetMemory();
    if (b_nonempty) { B->ResetMemory(); }
 }
@@ -5333,27 +5329,21 @@ static PetscErrorCode __mfem_pc_shell_destroy(PC pc)
    PetscFunctionReturn(PETSC_SUCCESS);
 }
 
-static PetscErrorCode __mfem_array_container_destroy(PetscCtxRt ptr)
+#if PETSC_VERSION_LT(3,23,0)
+
+static PetscErrorCode __mfem_array_container_destroy(void *ptr)
 {
    PetscErrorCode ierr;
 
    PetscFunctionBeginUser;
-#if PETSC_VERSION_LT(3,23,0)
    ierr = PetscFree(ptr); CHKERRQ(ierr);
-#else
-   ierr = PetscFree(*(void**)ptr); CHKERRQ(ierr);
-#endif
    PetscFunctionReturn(PETSC_SUCCESS);
 }
 
-static PetscErrorCode __mfem_matarray_container_destroy(PetscCtxRt ptr)
+static PetscErrorCode __mfem_matarray_container_destroy(void *ptr)
 {
-#if PETSC_VERSION_LT(3,23,0)
    mfem::Array<Mat> *a = (mfem::Array<Mat>*)ptr;
-#else
-   mfem::Array<Mat> *a = *(mfem::Array<Mat>**)ptr;
-#endif
-   PetscErrorCode ierr;
+   PetscErrorCode   ierr;
 
    PetscFunctionBeginUser;
    for (int i=0; i<a->Size(); i++)
@@ -5366,16 +5356,41 @@ static PetscErrorCode __mfem_matarray_container_destroy(PetscCtxRt ptr)
    PetscFunctionReturn(PETSC_SUCCESS);
 }
 
-#if PETSC_VERSION_LT(3,23,0)
-static PetscErrorCode __mfem_monitor_ctx_destroy(void **ctx)
 #else
-static PetscErrorCode __mfem_monitor_ctx_destroy(PetscCtxRt ctx)
-#endif
+
+static PetscErrorCode __mfem_array_container_destroy(void **ptr)
 {
    PetscErrorCode ierr;
 
    PetscFunctionBeginUser;
-   ierr = PetscFree(*(void**)ctx); CHKERRQ(ierr);
+   ierr = PetscFree(*ptr); CHKERRQ(ierr);
+   PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+static PetscErrorCode __mfem_matarray_container_destroy(void **ptr)
+{
+   mfem::Array<Mat> *a = (mfem::Array<Mat>*)*ptr;
+   PetscErrorCode   ierr;
+
+   PetscFunctionBeginUser;
+   for (int i=0; i<a->Size(); i++)
+   {
+      Mat M = (*a)[i];
+      MPI_Comm comm = PetscObjectComm((PetscObject)M);
+      ierr = MatDestroy(&M); CCHKERRQ(comm,ierr);
+   }
+   delete a;
+   PetscFunctionReturn(PETSC_SUCCESS);
+}
+
+#endif
+
+static PetscErrorCode __mfem_monitor_ctx_destroy(void **ctx)
+{
+   PetscErrorCode  ierr;
+
+   PetscFunctionBeginUser;
+   ierr = PetscFree(*ctx); CHKERRQ(ierr);
    PetscFunctionReturn(PETSC_SUCCESS);
 }
 
@@ -5620,7 +5635,11 @@ static PetscErrorCode MatConvert_hypreParCSR_AIJ(hypre_ParCSRMatrix* hA,Mat* pA)
 
       ierr = PetscContainerCreate(comm,&c); CHKERRQ(ierr);
       ierr = PetscContainerSetPointer(c,ptrs[i]); CHKERRQ(ierr);
+#if PETSC_VERSION_LT(3,23,0)
+      ierr = PetscContainerSetUserDestroy(c,__mfem_array_container_destroy);
+#else
       ierr = PetscContainerSetCtxDestroy(c,__mfem_array_container_destroy);
+#endif
       CHKERRQ(ierr);
       ierr = PetscObjectCompose((PetscObject)(*pA),names[i],(PetscObject)c);
       CHKERRQ(ierr);
@@ -5714,7 +5733,11 @@ static PetscErrorCode MatConvert_hypreParCSR_IS(hypre_ParCSRMatrix* hA,Mat* pA)
 
       ierr = PetscContainerCreate(PETSC_COMM_SELF,&c); CHKERRQ(ierr);
       ierr = PetscContainerSetPointer(c,ptrs[i]); CHKERRQ(ierr);
+#if PETSC_VERSION_LT(3,23,0)
+      ierr = PetscContainerSetUserDestroy(c,__mfem_array_container_destroy);
+#else
       ierr = PetscContainerSetCtxDestroy(c,__mfem_array_container_destroy);
+#endif
       CHKERRQ(ierr);
       ierr = PetscObjectCompose((PetscObject)lA,names[i],(PetscObject)c);
       CHKERRQ(ierr);

linalg/vector.cpp

@@ -107,7 +107,7 @@ static Array<DevicePair<real_t, real_t>> &Lpvector_workspace()
 
 Vector::Vector(const Vector &v)
 {
-   const int s = v.Size();
+   const bigint s = v.Size();
    size = s;
    if (s > 0)
    {
@@ -126,10 +126,8 @@ Vector::Vector(Vector &&v)
 
 void Vector::Load(std::istream **in, int np, int *dim)
 {
-   int i, j, s;
-
-   s = 0;
-   for (i = 0; i < np; i++)
+   bigint s = 0;
+   for (int i = 0; i < np; i++)
    {
       s += dim[i];
    }
@@ -137,10 +135,10 @@ void Vector::Load(std::istream **in, int np, int *dim)
    SetSize(s);
    HostWrite();
 
-   int p = 0;
-   for (i = 0; i < np; i++)
+   bigint p = 0;
+   for (int i = 0; i < np; i++)
    {
-      for (j = 0; j < dim[i]; j++)
+      for (int j = 0; j < dim[i]; j++)
       {
          *in[i] >> data[p++];
          // Clang's libc++ sets the failbit when (correctly) parsing subnormals,
@@ -153,12 +151,12 @@ void Vector::Load(std::istream **in, int np, int *dim)
    }
 }
 
-void Vector::Load(std::istream &in, int Size)
+void Vector::Load(std::istream &in, bigint Size)
 {
    SetSize(Size);
    HostWrite();
 
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       in >> data[i];
       // Clang's libc++ sets the failbit when (correctly) parsing subnormals,
@@ -170,12 +168,12 @@ void Vector::Load(std::istream &in, int Size)
    }
 }
 
-real_t &Vector::Elem(int i)
+real_t &Vector::Elem(bigint i)
 {
    return operator()(i);
 }
 
-const real_t &Vector::Elem(int i) const
+const real_t &Vector::Elem(bigint i) const
 {
    return operator()(i);
 }
@@ -187,7 +185,7 @@ real_t Vector::operator*(const real_t *v) const
 #ifdef MFEM_USE_LEGACY_OPENMP
    #pragma omp parallel for reduction(+:dot)
 #endif
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       dot += data[i] * v[i];
    }
@@ -232,18 +230,22 @@ Vector &Vector::operator=(Vector &&v)
 Vector &Vector::operator=(real_t value)
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = Write(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] = value; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] = value;
+   });
    return *this;
 }
 
 Vector &Vector::operator*=(real_t c)
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] *= c; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] *= c;
+   });
    return *this;
 }
 
@@ -252,20 +254,24 @@ Vector &Vector::operator*=(const Vector &v)
    MFEM_ASSERT(size == v.size, "incompatible Vectors!");
 
    const bool use_dev = UseDevice() || v.UseDevice();
-   const int N = size;
    const auto x = v.Read(use_dev);
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] *= x[i]; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] *= x[i];
+   });
    return *this;
 }
 
 Vector &Vector::operator/=(real_t c)
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    const real_t m = 1.0/c;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] *= m; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] *= m;
+   });
    return *this;
 }
 
@@ -274,19 +280,23 @@ Vector &Vector::operator/=(const Vector &v)
    MFEM_ASSERT(size == v.size, "incompatible Vectors!");
 
    const bool use_dev = UseDevice() || v.UseDevice();
-   const int N = size;
    const auto x = v.Read(use_dev);
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] /= x[i]; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] /= x[i];
+   });
    return *this;
 }
 
 Vector &Vector::operator-=(real_t c)
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] -= c; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] -= c;
+   });
    return *this;
 }
 
@@ -295,19 +305,23 @@ Vector &Vector::operator-=(const Vector &v)
    MFEM_ASSERT(size == v.size, "incompatible Vectors!");
 
    const bool use_dev = UseDevice() || v.UseDevice();
-   const int N = size;
    const auto x = v.Read(use_dev);
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] -= x[i]; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] -= x[i];
+   });
    return *this;
 }
 
 Vector &Vector::operator+=(real_t c)
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] += c; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] += c;
+   });
    return *this;
 }
 
@@ -316,10 +330,12 @@ Vector &Vector::operator+=(const Vector &v)
    MFEM_ASSERT(size == v.size, "incompatible Vectors!");
 
    const bool use_dev = UseDevice() || v.UseDevice();
-   const int N = size;
    const auto x = v.Read(use_dev);
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] += x[i]; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] += x[i];
+   });
    return *this;
 }
 
@@ -329,11 +345,13 @@ Vector &Vector::Add(const real_t a, const Vector &Va)
 
    if (a != 0.0)
    {
-      const int N = size;
       const bool use_dev = UseDevice() || Va.UseDevice();
       const auto x = Va.Read(use_dev);
       auto y = ReadWrite(use_dev);
-      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] += a * x[i]; });
+      mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+      {
+         y[i] += a * x[i];
+      });
    }
    return *this;
 }
@@ -343,58 +361,69 @@ Vector &Vector::Set(const real_t a, const Vector &Va)
    MFEM_ASSERT(size == Va.size, "incompatible Vectors!");
 
    const bool use_dev = UseDevice() || Va.UseDevice();
-   const int N = size;
    const auto x = Va.Read(use_dev);
    auto y = Write(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] = a * x[i]; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] = a * x[i];
+   });
    return *this;
 }
 
-void Vector::SetVector(const Vector &v, int offset)
+void Vector::SetVector(const Vector &v, bigint offset)
 {
    MFEM_ASSERT(v.Size() + offset <= size, "invalid sub-vector");
 
    const bool use_dev = UseDevice() || v.UseDevice();
-   const int vs = v.Size();
+   const bigint vs = v.Size();
    const auto vp = v.Read(use_dev);
    // Use read+write access for *this - we only modify some of its entries
    auto p = ReadWrite(use_dev) + offset;
-   mfem::forall_switch(use_dev, vs, [=] MFEM_HOST_DEVICE (int i) { p[i] = vp[i]; });
+   mfem::forall_switch(use_dev, vs, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      p[i] = vp[i];
+   });
 }
 
-void Vector::AddSubVector(const Vector &v, int offset)
+void Vector::AddSubVector(const Vector &v, bigint offset)
 {
    MFEM_ASSERT(v.Size() + offset <= size, "invalid sub-vector");
 
    const bool use_dev = UseDevice() || v.UseDevice();
-   const int vs = v.Size();
+   const bigint vs = v.Size();
    const auto vp = v.Read(use_dev);
    auto p = ReadWrite(use_dev) + offset;
-   mfem::forall_switch(use_dev, vs, [=] MFEM_HOST_DEVICE (int i) { p[i] += vp[i]; });
+   mfem::forall_switch(use_dev, vs, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      p[i] += vp[i];
+   });
 }
 
 void Vector::Neg()
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] = -y[i]; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] = -y[i];
+   });
 }
 
 void Vector::Reciprocal()
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] = 1.0/y[i]; });
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] = 1.0/y[i];
+   });
 }
 
 void Vector::Abs()
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
    {
       y[i] = std::abs(y[i]);
    });
@@ -403,9 +432,8 @@ void Vector::Abs()
 void Vector::Pow(const real_t p)
 {
    const bool use_dev = UseDevice();
-   const int N = size;
    auto y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE (bigint i)
    {
       y[i] = std::pow(y[i], p);
    });
@@ -418,15 +446,18 @@ void add(const Vector &v1, const Vector &v2, Vector &v)
 
 #if !defined(MFEM_USE_LEGACY_OPENMP)
    const bool use_dev = v1.UseDevice() || v2.UseDevice() || v.UseDevice();
-   const int N = v.size;
+   const bigint N = v.size;
    // Note: get read access first, in case v is the same as v1/v2.
    const auto x1 = v1.Read(use_dev);
    const auto x2 = v2.Read(use_dev);
    auto y = v.Write(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { y[i] = x1[i] + x2[i]; });
+   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      y[i] = x1[i] + x2[i];
+   });
 #else
    #pragma omp parallel for
-   for (int i = 0; i < v.size; i++)
+   for (bigint i = 0; i < v.size; i++)
    {
       v.data[i] = v1.data[i] + v2.data[i];
    }
@@ -450,21 +481,21 @@ void add(const Vector &v1, real_t alpha, const Vector &v2, Vector &v)
    {
 #if !defined(MFEM_USE_LEGACY_OPENMP)
       const bool use_dev = v1.UseDevice() || v2.UseDevice() || v.UseDevice();
-      const int N = v.size;
+      const bigint N = v.size;
       // Note: get read access first, in case v is the same as v1/v2.
       const auto d_x = v1.Read(use_dev);
       const auto d_y = v2.Read(use_dev);
       auto d_z = v.Write(use_dev);
-      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
       {
          d_z[i] = d_x[i] + alpha * d_y[i];
       });
 #else
       const real_t *v1p = v1.data, *v2p = v2.data;
       real_t *vp = v.data;
-      const int s = v.size;
+      const bigint s = v.size;
       #pragma omp parallel for
-      for (int i = 0; i < s; i++)
+      for (bigint i = 0; i < s; i++)
       {
          vp[i] = v1p[i] + alpha*v2p[i];
       }
@@ -489,12 +520,12 @@ void add(const real_t a, const Vector &x, const Vector &y, Vector &z)
    {
 #if !defined(MFEM_USE_LEGACY_OPENMP)
       const bool use_dev = x.UseDevice() || y.UseDevice() || z.UseDevice();
-      const int N = x.size;
+      const bigint N = x.size;
       // Note: get read access first, in case z is the same as x/y.
       const auto xd = x.Read(use_dev);
       const auto yd = y.Read(use_dev);
       auto zd = z.Write(use_dev);
-      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
       {
          zd[i] = a * (xd[i] + yd[i]);
       });
@@ -502,9 +533,9 @@ void add(const real_t a, const Vector &x, const Vector &y, Vector &z)
       const real_t *xp = x.data;
       const real_t *yp = y.data;
       real_t       *zp = z.data;
-      const int      s = x.size;
+      const bigint   s = x.size;
       #pragma omp parallel for
-      for (int i = 0; i < s; i++)
+      for (bigint i = 0; i < s; i++)
       {
          zp[i] = a * (xp[i] + yp[i]);
       }
@@ -544,12 +575,12 @@ void add(const real_t a, const Vector &x,
    {
 #if !defined(MFEM_USE_LEGACY_OPENMP)
       const bool use_dev = x.UseDevice() || y.UseDevice() || z.UseDevice();
-      const int N = x.size;
+      const bigint N = x.size;
       // Note: get read access first, in case z is the same as x/y.
       const auto xd = x.Read(use_dev);
       const auto yd = y.Read(use_dev);
       auto zd = z.Write(use_dev);
-      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
       {
          zd[i] = a * xd[i] + b * yd[i];
       });
@@ -557,9 +588,9 @@ void add(const real_t a, const Vector &x,
       const real_t *xp = x.data;
       const real_t *yp = y.data;
       real_t       *zp = z.data;
-      const int      s = x.size;
+      const bigint   s = x.size;
       #pragma omp parallel for
-      for (int i = 0; i < s; i++)
+      for (bigint i = 0; i < s; i++)
       {
          zp[i] = a * xp[i] + b * yp[i];
       }
@@ -574,12 +605,12 @@ void subtract(const Vector &x, const Vector &y, Vector &z)
 
 #if !defined(MFEM_USE_LEGACY_OPENMP)
    const bool use_dev = x.UseDevice() || y.UseDevice() || z.UseDevice();
-   const int N = x.size;
+   const bigint N = x.size;
    // Note: get read access first, in case z is the same as x/y.
    const auto xd = x.Read(use_dev);
    const auto yd = y.Read(use_dev);
    auto zd = z.Write(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
    {
       zd[i] = xd[i] - yd[i];
    });
@@ -587,9 +618,9 @@ void subtract(const Vector &x, const Vector &y, Vector &z)
    const real_t *xp = x.data;
    const real_t *yp = y.data;
    real_t       *zp = z.data;
-   const int     s = x.size;
+   const bigint   s = x.size;
    #pragma omp parallel for
-   for (int i = 0; i < s; i++)
+   for (bigint i = 0; i < s; i++)
    {
       zp[i] = xp[i] - yp[i];
    }
@@ -613,12 +644,12 @@ void subtract(const real_t a, const Vector &x, const Vector &y, Vector &z)
    {
 #if !defined(MFEM_USE_LEGACY_OPENMP)
       const bool use_dev = x.UseDevice() || y.UseDevice() || z.UseDevice();
-      const int N = x.size;
+      const bigint N = x.size;
       // Note: get read access first, in case z is the same as x/y.
       const auto xd = x.Read(use_dev);
       const auto yd = y.Read(use_dev);
       auto zd = z.Write(use_dev);
-      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+      mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
       {
          zd[i] = a * (xd[i] - yd[i]);
       });
@@ -626,9 +657,9 @@ void subtract(const real_t a, const Vector &x, const Vector &y, Vector &z)
       const real_t *xp = x.data;
       const real_t *yp = y.data;
       real_t       *zp = z.data;
-      const int      s = x.size;
+      const bigint   s = x.size;
       #pragma omp parallel for
-      for (int i = 0; i < s; i++)
+      for (bigint i = 0; i < s; i++)
       {
          zp[i] = a * (xp[i] - yp[i]);
       }
@@ -655,12 +686,12 @@ void Vector::median(const Vector &lo, const Vector &hi)
                "incompatible Vectors!");
 
    const bool use_dev = UseDevice() || lo.UseDevice() || hi.UseDevice();
-   const int N = size;
+   const bigint N = size;
    // Note: get read access first, in case *this is the same as lo/hi.
    const auto l = lo.Read(use_dev);
    const auto h = hi.Read(use_dev);
    auto m = Write(use_dev);
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
    {
       if (m[i] < l[i])
       {
@@ -675,13 +706,13 @@ void Vector::median(const Vector &lo, const Vector &hi)
 
 void Vector::GetSubVector(const Array<int> &dofs, Vector &elemvect) const
 {
-   const int n = dofs.Size();
+   const bigint n = dofs.Size();
    elemvect.SetSize(n);
    const bool use_dev = dofs.UseDevice() || elemvect.UseDevice();
    const auto d_X = Read(use_dev);
    const auto d_dofs = dofs.Read(use_dev);
    auto d_y = elemvect.Write(use_dev);
-   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (bigint i)
    {
       const int dof_i = d_dofs[i];
       d_y[i] = dof_i >= 0 ? d_X[dof_i] : -d_X[-dof_i-1];
@@ -691,8 +722,8 @@ void Vector::GetSubVector(const Array<int> &dofs, Vector &elemvect) const
 void Vector::GetSubVector(const Array<int> &dofs, real_t *elem_data) const
 {
    HostRead();
-   const int n = dofs.Size();
-   for (int i = 0; i < n; i++)
+   const bigint n = dofs.Size();
+   for (bigint i = 0; i < n; i++)
    {
       const int j = dofs[i];
       elem_data[i] = (j >= 0) ? data[j] : -data[-1-j];
@@ -702,11 +733,11 @@ void Vector::GetSubVector(const Array<int> &dofs, real_t *elem_data) const
 void Vector::SetSubVector(const Array<int> &dofs, const real_t value)
 {
    const bool use_dev = UseDevice() || dofs.UseDevice();
-   const int n = dofs.Size();
+   const bigint n = dofs.Size();
    // Use read+write access for *this - we only modify some of its entries
    auto d_X = ReadWrite(use_dev);
    const auto d_dofs = dofs.Read(use_dev);
-   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (bigint i)
    {
       const int j = d_dofs[i];
       if (j >= 0)
@@ -723,7 +754,7 @@ void Vector::SetSubVector(const Array<int> &dofs, const real_t value)
 void Vector::SetSubVectorHost(const Array<int> &dofs, const real_t value)
 {
    HostReadWrite();
-   for (int i = 0; i < dofs.Size(); ++i)
+   for (bigint i = 0; i < dofs.Size(); ++i)
    {
       const int j = dofs[i];
       if (j >= 0)
@@ -744,12 +775,12 @@ void Vector::SetSubVector(const Array<int> &dofs, const Vector &elemvect)
                << ", length of elemvect is " << elemvect.Size());
 
    const bool use_dev = dofs.UseDevice() || elemvect.UseDevice();
-   const int n = dofs.Size();
+   const bigint n = dofs.Size();
    // Use read+write access for X - we only modify some of its entries
    auto d_X = ReadWrite(use_dev);
    const auto d_y = elemvect.Read(use_dev);
    const auto d_dofs = dofs.Read(use_dev);
-   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (bigint i)
    {
       const int dof_i = d_dofs[i];
       if (dof_i >= 0)
@@ -767,10 +798,10 @@ void Vector::SetSubVector(const Array<int> &dofs, real_t *elem_data)
 {
    // Use read+write access because we overwrite only part of the data.
    HostReadWrite();
-   const int n = dofs.Size();
-   for (int i = 0; i < n; i++)
+   const bigint n = dofs.Size();
+   for (bigint i = 0; i < n; i++)
    {
-      const int j= dofs[i];
+      const int j = dofs[i];
       if (j >= 0)
       {
          operator()(j) = elem_data[i];
@@ -789,11 +820,11 @@ void Vector::AddElementVector(const Array<int> &dofs, const Vector &elemvect)
                ", length of elemvect is " << elemvect.Size());
 
    const bool use_dev = dofs.UseDevice() || elemvect.UseDevice();
-   const int n = dofs.Size();
+   const bigint n = dofs.Size();
    const auto d_y = elemvect.Read(use_dev);
    const auto d_dofs = dofs.Read(use_dev);
    auto d_X = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (bigint i)
    {
       const int j = d_dofs[i];
       if (j >= 0)
@@ -810,8 +841,8 @@ void Vector::AddElementVector(const Array<int> &dofs, const Vector &elemvect)
 void Vector::AddElementVector(const Array<int> &dofs, real_t *elem_data)
 {
    HostReadWrite();
-   const int n = dofs.Size();
-   for (int i = 0; i < n; i++)
+   const bigint n = dofs.Size();
+   for (bigint i = 0; i < n; i++)
    {
       const int j = dofs[i];
       if (j >= 0)
@@ -833,11 +864,11 @@ void Vector::AddElementVector(const Array<int> &dofs, const real_t a,
                ", length of elemvect is " << elemvect.Size());
 
    const bool use_dev = dofs.UseDevice() || elemvect.UseDevice();
-   const int n = dofs.Size();
+   const bigint n = dofs.Size();
    const auto d_x = elemvect.Read(use_dev);
    const auto d_dofs = dofs.Read(use_dev);
    auto d_y = ReadWrite(use_dev);
-   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (int i)
+   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (bigint i)
    {
       const int j = d_dofs[i];
       if (j >= 0)
@@ -854,17 +885,26 @@ void Vector::AddElementVector(const Array<int> &dofs, const real_t a,
 void Vector::SetSubVectorComplement(const Array<int> &dofs, const real_t val)
 {
    const bool use_dev = UseDevice() || dofs.UseDevice();
-   const int n = dofs.Size();
-   const int N = size;
+   const bigint n = dofs.Size();
+   const bigint N = size;
    Vector dofs_vals(n, use_dev ?
                     Device::GetDeviceMemoryType() :
                     Device::GetHostMemoryType());
    auto d_data = ReadWrite(use_dev);
    auto d_dofs_vals = dofs_vals.Write(use_dev);
    const auto d_dofs = dofs.Read(use_dev);
-   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (int i) { d_dofs_vals[i] = d_data[d_dofs[i]]; });
-   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (int i) { d_data[i] = val; });
-   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (int i) { d_data[d_dofs[i]] = d_dofs_vals[i]; });
+   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      d_dofs_vals[i] = d_data[d_dofs[i]];
+   });
+   mfem::forall_switch(use_dev, N, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      d_data[i] = val;
+   });
+   mfem::forall_switch(use_dev, n, [=] MFEM_HOST_DEVICE (bigint i)
+   {
+      d_data[d_dofs[i]] = d_dofs_vals[i];
+   });
 }
 
 void Vector::Print(std::ostream &os, int width) const
@@ -903,7 +943,6 @@ void Vector::Print(adios2stream &os,
 
 void Vector::Print_HYPRE(std::ostream &os) const
 {
-   int i;
    std::ios::fmtflags old_fmt = os.flags();
    os.setf(std::ios::scientific);
    std::streamsize old_prec = os.precision(14);
@@ -911,7 +950,7 @@ void Vector::Print_HYPRE(std::ostream &os) const
    os << size << '\n';  // number of rows
 
    data.Read(MemoryClass::HOST, size);
-   for (i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       os << ZeroSubnormal(data[i]) << '\n';
    }
@@ -931,7 +970,7 @@ void Vector::PrintMathematica(std::ostream & os) const
    os << "{\n";
 
    data.Read(MemoryClass::HOST, size);
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       os << "Internal`StringToMReal[\"" << ZeroSubnormal(data[i]) << "\"]";
       if (i < size - 1) { os << ','; }
@@ -959,7 +998,7 @@ void Vector::Randomize(int seed)
    srand((unsigned)seed);
 
    HostWrite();
-   for (int i = 0; i < size; i++)
+   for (bigint i = 0; i < size; i++)
    {
       data[i] = rand_real();
    }
@@ -978,6 +1017,7 @@ real_t Vector::Norml2() const
    res.first = 0;
    res.second = 0;
    // first compute sum (|m_data|/scale)^2
+   // FIXME: bigint support
    reduce(size, res, [=] MFEM_HOST_DEVICE(int i, value_type &r)
    {
       real_t n = fabs(m_data[i]);
@@ -1007,6 +1047,7 @@ real_t Vector::Normlinf() const
 
    real_t res = 0;
    const auto m_data = Read(UseDevice());
+   // FIXME: bigint support
    reduce(size, res, [=] MFEM_HOST_DEVICE(int i, real_t &r)
    {
       r = fmax(r, fabs(m_data[i]));
@@ -1021,6 +1062,7 @@ real_t Vector::Norml1() const
 
    real_t res = 0;
    const auto m_data = Read(UseDevice());
+   // FIXME: bigint support
    reduce(size, res, [=] MFEM_HOST_DEVICE(int i, real_t &r)
    {
       r += fabs(m_data[i]);
@@ -1050,6 +1092,7 @@ real_t Vector::Normlp(real_t p) const
       res.second = 0;
       const auto m_data = Read(UseDevice());
       // first compute sum (|m_data|/scale)^p
+      // FIXME: bigint support
       reduce(size, res, [=] MFEM_HOST_DEVICE(int i, value_type &r)
       {
          real_t n = fabs(m_data[i]);
@@ -1097,6 +1140,7 @@ real_t Vector::operator*(const Vector &v) const
    const auto compute_dot = [&]()
    {
       real_t res = 0;
+      // FIXME: bigint support
       reduce(size, res, [=] MFEM_HOST_DEVICE (int i, real_t &r)
       {
          r += m_data[i] * v_data[i];
@@ -1122,11 +1166,11 @@ real_t Vector::operator*(const Vector &v) const
          #pragma omp master
          th_dot.SetSize(nt);
          const int tid = omp_get_thread_num();
-         const int stride = (size + nt - 1) / nt;
-         const int start = tid * stride;
-         const int stop = std::min(start + stride, size);
+         const bigint stride = (size + nt - 1) / nt;
+         const bigint start = tid * stride;
+         const bigint stop = std::min(start + stride, size);
          real_t my_dot = 0.0;
-         for (int i = start; i < stop; i++)
+         for (bigint i = start; i < stop; i++)
          {
             my_dot += m_data[i] * v_data[i];
          }
@@ -1138,7 +1182,7 @@ real_t Vector::operator*(const Vector &v) const
       // The standard way of computing the dot product is non-deterministic
       real_t prod = 0.0;
       #pragma omp parallel for reduction(+ : prod)
-      for (int i = 0; i < size; i++)
+      for (bigint i = 0; i < size; i++)
       {
          prod += m_data[i] * v_data[i];
       }
@@ -1168,6 +1212,7 @@ real_t Vector::Min() const
    const auto compute_min = [&]()
    {
       real_t res = infinity();
+      // FIXME: bigint support
       reduce(size, res, [=] MFEM_HOST_DEVICE(int i, real_t &r)
       {
          r = fmin(r, m_data[i]);
@@ -1185,7 +1230,7 @@ real_t Vector::Min() const
    {
       real_t minimum = m_data[0];
       #pragma omp parallel for reduction(min:minimum)
-      for (int i = 0; i < size; i++)
+      for (bigint i = 0; i < size; i++)
       {
          minimum = std::min(minimum, m_data[i]);
       }
@@ -1214,6 +1259,7 @@ real_t Vector::Max() const
    const auto compute_max = [&]()
    {
       real_t res = -infinity();
+      // FIXME: bigint support
       reduce(size, res, [=] MFEM_HOST_DEVICE(int i, real_t &r)
       {
          r = fmax(r, m_data[i]);
@@ -1231,7 +1277,7 @@ real_t Vector::Max() const
    {
       real_t maximum = m_data[0];
       #pragma omp parallel for reduction(max : maximum)
-      for (int i = 0; i < size; i++)
+      for (bigint i = 0; i < size; i++)
       {
          maximum = fmax(maximum, m_data[i]);
       }
@@ -1249,6 +1295,7 @@ real_t Vector::Sum() const
 
    real_t res = 0;
    const auto m_data = Read(UseDevice());
+   // FIXME: bigint support
    reduce(size, res, [=] MFEM_HOST_DEVICE(int i, real_t &r)
    {
       r += m_data[i];
@@ -1266,10 +1313,13 @@ void Vector::DeleteAt(const Array<int> &indices)
       // extra entry for number of selected out
       Array<int> workspace(size + 1);
       const auto d_flag = workspace.Write(use_dev);
-      mfem::forall_switch(use_dev, size,
-      [=] MFEM_HOST_DEVICE(int i) { d_flag[i] = true; });
+      mfem::forall_switch(use_dev, size, [=] MFEM_HOST_DEVICE(bigint i)
+      {
+         d_flag[i] = true;
+      });
       const auto d_indices = indices.Read(use_dev);
-      mfem::forall_switch(use_dev, indices.Size(), [=] MFEM_HOST_DEVICE(int i)
+      mfem::forall_switch(
+         use_dev, indices.Size(), [=] MFEM_HOST_DEVICE(bigint i)
       {
          // fine as long as indices are unique; to support non-unique indices
          // assignment to d_flag must be atomic

linalg/vector.hpp

@@ -40,7 +40,7 @@ namespace mfem
 
 /** Count the number of entries in an array of doubles for which isfinite
     is false, i.e. the entry is a NaN or +/-Inf. */
-inline int CheckFinite(const real_t *v, const int n);
+inline bigint CheckFinite(const real_t *v, const bigint n);
 
 /// Define a shortcut for std::numeric_limits<double>::infinity()
 #ifndef __CYGWIN__
@@ -83,7 +83,7 @@ class Vector
 protected:
 
    Memory<real_t> data;
-   int size;
+   bigint size;
 
 public:
 
@@ -99,30 +99,30 @@ public:
 
    /// @brief Creates vector of size s.
    /// @warning Entries are not initialized to zero!
-   explicit Vector(int s);
+   explicit Vector(bigint s);
 
    /// Creates a vector referencing an array of doubles, owned by someone else.
    /** The pointer @a data_ can be NULL. The data array can be replaced later
        with SetData(). */
-   Vector(real_t *data_, int size_)
+   Vector(real_t *data_, bigint size_)
    { data.Wrap(data_, size_, false); size = size_; }
 
    /** @brief Create a Vector referencing a sub-vector of the Vector @a base
        starting at the given offset, @a base_offset, and size @a size_. */
-   Vector(Vector &base, int base_offset, int size_)
+   Vector(Vector &base, bigint base_offset, bigint size_)
       : data(base.data, base_offset, size_), size(size_) { }
 
    /// Create a Vector of size @a size_ using MemoryType @a mt.
-   Vector(int size_, MemoryType mt)
+   Vector(bigint size_, MemoryType mt)
       : data(size_, mt), size(size_) { }
 
    /** @brief Create a Vector of size @a size_ using host MemoryType @a h_mt and
        device MemoryType @a d_mt. */
-   Vector(int size_, MemoryType h_mt, MemoryType d_mt)
+   Vector(bigint size_, MemoryType h_mt, MemoryType d_mt)
       : data(size_, h_mt, d_mt), size(size_) { }
 
    /// Create a vector from a statically sized C-style array of convertible type
-   template <typename CT, int N>
+   template <typename CT, bigint N>
    explicit Vector(const CT (&values)[N]) : Vector(N)
    { std::copy(values, values + N, begin()); }
 
@@ -130,7 +130,7 @@ public:
    template <typename CT, typename std::enable_if<
                 std::is_convertible<CT,real_t>::value,bool>::type = true>
    explicit Vector(std::initializer_list<CT> values) :
-      Vector(static_cast<int> (values.size()))
+      Vector(static_cast<bigint> (values.size()))
    { std::copy(values.begin(), values.end(), begin()); }
 
    /// Enable execution of Vector operations using the mfem::Device.
@@ -151,10 +151,10 @@ public:
    void Load(std::istream ** in, int np, int * dim);
 
    /// Load a vector from an input stream.
-   void Load(std::istream &in, int Size);
+   void Load(std::istream &in, bigint Size);
 
    /// Load a vector from an input stream, reading the size from the stream.
-   void Load(std::istream &in) { int s; in >> s; Load(in, s); }
+   void Load(std::istream &in) { bigint s; in >> s; Load(in, s); }
 
    /// @brief Resize the vector to size @a s.
    /** If the new size is less than or equal to Capacity() then the internal
@@ -164,16 +164,18 @@ public:
        @warning In the second case above (new size greater than current one),
        the vector will allocate new data array, even if it did not own the
        original data! Also, new entries are not initialized! */
-   void SetSize(int s);
+   void SetSize(bigint s);
 
    /// Resize the vector to size @a s using MemoryType @a mt.
-   void SetSize(int s, MemoryType mt);
+   void SetSize(bigint s, MemoryType mt);
 
    /// Resize the vector to size @a s using the MemoryType of @a v.
-   void SetSize(int s, const Vector &v) { SetSize(s, v.GetMemory().GetMemoryType()); }
+   void SetSize(bigint s, const Vector &v)
+   { SetSize(s, v.GetMemory().GetMemoryType()); }
 
-   /// Update \ref Capacity() to @a res (if less than current), keeping existing entries.
-   void Reserve(int res);
+   /** @brief Update \ref Capacity() to @a res (if less than current), keeping
+       existing entries. */
+   void Reserve(bigint res);
 
    /// Delete entries at @a indices and resize vector accordingly.
    /// @warning Indices must be unique!
@@ -188,13 +190,14 @@ public:
        also used as the new Capacity().
        @warning This method should be called only when OwnsData() is false.
        @sa NewDataAndSize(). */
-   void SetDataAndSize(real_t *d, int s) { data.Wrap(d, s, false); size = s; }
+   void SetDataAndSize(real_t *d, bigint s)
+   { data.Wrap(d, s, false); size = s; }
 
    /// Set the Vector data and size, deleting the old data, if owned.
    /** The Vector does not assume ownership of the new data. The new size is
        also used as the new Capacity().
        @sa SetDataAndSize(). */
-   void NewDataAndSize(real_t *d, int s)
+   void NewDataAndSize(real_t *d, bigint s)
    {
       data.Delete();
       SetDataAndSize(d, s);
@@ -209,14 +212,15 @@ public:
        the Vector object takes ownership of all pointers owned by @a mem.
 
        @sa NewDataAndSize(). */
-   inline void NewMemoryAndSize(const Memory<real_t> &mem, int s, bool own_mem);
+   inline void NewMemoryAndSize(const Memory<real_t> &mem, bigint s,
+                                bool own_mem);
 
    /// Reset the Vector to be a reference to a sub-vector of @a base.
-   inline void MakeRef(Vector &base, int offset, int size);
+   inline void MakeRef(Vector &base, bigint offset, bigint size);
 
    /** @brief Reset the Vector to be a reference to a sub-vector of @a base
        without changing its current size. */
-   inline void MakeRef(Vector &base, int offset);
+   inline void MakeRef(Vector &base, bigint offset);
 
    /// Set the Vector data (host pointer) ownership flag.
    void MakeDataOwner() const { data.SetHostPtrOwner(true); }
@@ -231,11 +235,11 @@ public:
    { data.DeleteDevice(copy_to_host); }
 
    /// Returns the size of the vector.
-   inline int Size() const { return size; }
+   inline bigint Size() const { return size; }
 
    /// Return the size of the currently allocated data array.
    /** It is always true that Capacity() >= Size(). */
-   inline int Capacity() const { return data.Capacity(); }
+   inline bigint Capacity() const { return data.Capacity(); }
 
    /// Return a pointer to the beginning of the Vector data.
    /** @warning This method should be used with caution as it gives write access
@@ -286,26 +290,26 @@ public:
    inline real_t *StealData() { real_t *p; StealData(&p); return p; }
 
    /// Access Vector entries. Index i = 0 .. size-1.
-   real_t &Elem(int i);
+   real_t &Elem(bigint i);
 
    /// Read only access to Vector entries. Index i = 0 .. size-1.
-   const real_t &Elem(int i) const;
+   const real_t &Elem(bigint i) const;
 
    /// Access Vector entries using () for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline real_t &operator()(int i);
+   inline real_t &operator()(bigint i);
 
    /// Read only access to Vector entries using () for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline const real_t &operator()(int i) const;
+   inline const real_t &operator()(bigint i) const;
 
    /// Access Vector entries using [] for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline real_t &operator[](int i) { return (*this)(i); }
+   inline real_t &operator[](bigint i) { return (*this)(i); }
 
    /// Read only access to Vector entries using [] for 0-based indexing.
    /** @note If MFEM_DEBUG is enabled, bounds checking is performed. */
-   inline const real_t &operator[](int i) const { return (*this)(i); }
+   inline const real_t &operator[](bigint i) const { return (*this)(i); }
 
    /// Dot product with a `double *` array.
    /// This function always executes on the CPU. A HostRead() will be called if
@@ -357,10 +361,10 @@ public:
    Vector &Set(const real_t a, const Vector &x);
 
    /// (*this)[i + offset] = v[i]
-   void SetVector(const Vector &v, int offset);
+   void SetVector(const Vector &v, bigint offset);
 
    /// (*this)[i + offset] += v[i]
-   void AddSubVector(const Vector &v, int offset);
+   void AddSubVector(const Vector &v, bigint offset);
 
    /// (*this) = -(*this)
    void Neg();
@@ -511,7 +515,7 @@ public:
 
    /** @brief Count the number of entries in the Vector for which isfinite
        is false, i.e. the entry is a NaN or +/-Inf. */
-   int CheckFinite() const { return mfem::CheckFinite(HostRead(), size); }
+   bigint CheckFinite() const { return mfem::CheckFinite(HostRead(), size); }
 
    /// Destroys vector.
    virtual ~Vector();
@@ -561,17 +565,17 @@ inline bool IsFinite(const real_t &val)
 #endif
 }
 
-inline int CheckFinite(const real_t *v, const int n)
+inline bigint CheckFinite(const real_t *v, const bigint n)
 {
-   int bad = 0;
-   for (int i = 0; i < n; i++)
+   bigint bad = 0;
+   for (bigint i = 0; i < n; i++)
    {
       if (!IsFinite(v[i])) { bad++; }
    }
    return bad;
 }
 
-inline Vector::Vector(int s)
+inline Vector::Vector(bigint s)
 {
    MFEM_ASSERT(s>=0,"Unexpected negative size.");
    size = s;
@@ -581,7 +585,7 @@ inline Vector::Vector(int s)
    }
 }
 
-inline void Vector::SetSize(int s)
+inline void Vector::SetSize(bigint s)
 {
    if (s == size)
    {
@@ -601,7 +605,7 @@ inline void Vector::SetSize(int s)
    data.UseDevice(use_dev);
 }
 
-inline void Vector::SetSize(int s, MemoryType mt)
+inline void Vector::SetSize(bigint s, MemoryType mt)
 {
    if (mt == data.GetMemoryType())
    {
@@ -630,7 +634,7 @@ inline void Vector::SetSize(int s, MemoryType mt)
    data.UseDevice(use_dev);
 }
 
-inline void Vector::Reserve(int res)
+inline void Vector::Reserve(bigint res)
 {
    if (res > Capacity())
    {
@@ -642,7 +646,7 @@ inline void Vector::Reserve(int res)
    }
 }
 
-inline void Vector::NewMemoryAndSize(const Memory<real_t> &mem, int s,
+inline void Vector::NewMemoryAndSize(const Memory<real_t> &mem, bigint s,
                                      bool own_mem)
 {
    data.Delete();
@@ -657,14 +661,14 @@ inline void Vector::NewMemoryAndSize(const Memory<real_t> &mem, int s,
    }
 }
 
-inline void Vector::MakeRef(Vector &base, int offset, int s)
+inline void Vector::MakeRef(Vector &base, bigint offset, bigint s)
 {
    data.Delete();
    size = s;
    data.MakeAlias(base.GetMemory(), offset, s);
 }
 
-inline void Vector::MakeRef(Vector &base, int offset)
+inline void Vector::MakeRef(Vector &base, bigint offset)
 {
    data.Delete();
    data.MakeAlias(base.GetMemory(), offset, size);
@@ -678,7 +682,7 @@ inline void Vector::Destroy()
    data.UseDevice(use_dev);
 }
 
-inline real_t &Vector::operator()(int i)
+inline real_t &Vector::operator()(bigint i)
 {
    MFEM_ASSERT(data && i >= 0 && i < size,
                "index [" << i << "] is out of range [0," << size << ")");
@@ -686,7 +690,7 @@ inline real_t &Vector::operator()(int i)
    return data[i];
 }
 
-inline const real_t &Vector::operator()(int i) const
+inline const real_t &Vector::operator()(bigint i) const
 {
    MFEM_ASSERT(data && i >= 0 && i < size,
                "index [" << i << "] is out of range [0," << size << ")");
@@ -712,11 +716,11 @@ inline Vector::~Vector()
    data.Delete();
 }
 
-inline real_t DistanceSquared(const real_t *x, const real_t *y, const int n)
+inline real_t DistanceSquared(const real_t *x, const real_t *y, const bigint n)
 {
    real_t d = 0.0;
 
-   for (int i = 0; i < n; i++)
+   for (bigint i = 0; i < n; i++)
    {
       d += (x[i]-y[i])*(x[i]-y[i]);
    }
@@ -724,7 +728,7 @@ inline real_t DistanceSquared(const real_t *x, const real_t *y, const int n)
    return d;
 }
 
-inline real_t Distance(const real_t *x, const real_t *y, const int n)
+inline real_t Distance(const real_t *x, const real_t *y, const bigint n)
 {
    return std::sqrt(DistanceSquared(x, y, n));
 }

makefile

@@ -123,20 +123,15 @@ EXAMPLE_SUBDIRS = amgx caliper ginkgo hiop petsc pumi sundials superlu moonolith
 EXAMPLE_DIRS := examples $(addprefix examples/,$(EXAMPLE_SUBDIRS))
 EXAMPLE_TEST_DIRS := examples
 
-MINIAPP_ALL_SUBDIRS = common electromagnetics meshing performance tools \
+MINIAPP_SUBDIRS = common electromagnetics meshing performance tools \
  toys nurbs gslib adjoint solvers shifted mtop parelag tribol autodiff dfem \
  hooke multidomain dpg hdiv-linear-solver spde diag-smoothers contact \
- fluids/navier fluids/schrodinger-flow plasma plasma/pic
-MINIAPP_RECURSIVE_SUBDIRS = plasma/pic
-MINIAPP_SUBDIRS := $(filter-out \
-   $(MINIAPP_RECURSIVE_SUBDIRS),$(MINIAPP_ALL_SUBDIRS))
-MINIAPP_ALL_DIRS := $(addprefix miniapps/,$(MINIAPP_ALL_SUBDIRS))
+ fluids/navier fluids/schrodinger-flow
 MINIAPP_DIRS := $(addprefix miniapps/,$(MINIAPP_SUBDIRS))
 MINIAPP_TEST_DIRS := $(filter-out %/common,$(MINIAPP_DIRS))
 MINIAPP_USE_COMMON := $(addprefix miniapps/,electromagnetics meshing tools \
- toys gslib shifted dpg diag-smoothers fluids/navier plasma plasma/pic)
+ toys shifted dpg diag-smoothers fluids/navier)
 
-EM_ALL_DIRS = $(EXAMPLE_DIRS) $(MINIAPP_ALL_DIRS)
 EM_DIRS = $(EXAMPLE_DIRS) $(MINIAPP_DIRS)
 
 TEST_SUBDIRS = unit
@@ -151,7 +146,7 @@ MFEM_BUILD_DIR ?= .
 BUILD_DIR := $(MFEM_BUILD_DIR)
 BUILD_REAL_DIR := $(abspath $(BUILD_DIR))
 ifneq ($(BUILD_REAL_DIR),$(MFEM_REAL_DIR))
-   BUILD_SUBDIRS = $(DIRS) config $(EM_ALL_DIRS) doc $(TEST_DIRS)
+   BUILD_SUBDIRS = $(DIRS) config $(EM_DIRS) doc $(TEST_DIRS)
    CONFIG_FILE_DEF = -DMFEM_CONFIG_FILE='"$(BUILD_REAL_DIR)/config/_config.hpp"'
    BLD := $(if $(BUILD_REAL_DIR:$(CURDIR)=),$(BUILD_DIR)/,)
    $(if $(word 2,$(BLD)),$(error Spaces in BLD = "$(BLD)" are not supported))
@@ -488,10 +483,10 @@ $(OBJECT_FILES): $(BLD)%.o: $(SRC)%.cpp $(CONFIG_MK)
 
 all: examples miniapps $(TEST_DIRS)
 
-.PHONY: miniapps $(EM_ALL_DIRS) $(TEST_DIRS)
+.PHONY: miniapps $(EM_DIRS) $(TEST_DIRS)
 miniapps: $(MINIAPP_DIRS)
 $(MINIAPP_USE_COMMON): miniapps/common
-$(EM_ALL_DIRS) $(TEST_DIRS): lib
+$(EM_DIRS) $(TEST_DIRS): lib
 	$(MAKE) -C $(BLD)$(@)
 
 .PHONY: doc
@@ -699,7 +694,7 @@ local-config:
 .PHONY: build-config
 build-config:
 	for d in $(BUILD_SUBDIRS); do mkdir -p $(BLD)$${d}; done
-	for dir in "" $(addsuffix /,config $(EM_ALL_DIRS) doc $(TEST_DIRS)); do\
+	for dir in "" $(addsuffix /,config $(EM_DIRS) doc $(TEST_DIRS)); do \
 	   printf "# Auto-generated file.\n%s\n%s\n" \
 	      "MFEM_DIR = $(MFEM_REAL_DIR)" \
 	      "include \$$(MFEM_DIR)/$${dir}makefile" \
@@ -801,15 +796,13 @@ status info:
 
 ASTYLE = $(ASTYLE_BIN) --options=$(SRC)config/mfem.astylerc
 ASTYLE_VER = "Artistic Style Version 3.1"
-FORMAT_FILES = $(foreach dir,$(DIRS) $(EM_ALL_DIRS) config,$(dir)/*.?pp)
+FORMAT_FILES = $(foreach dir,$(DIRS) $(EM_DIRS) config,$(dir)/*.?pp)
 TESTS_SUBDIRS = unit benchmarks convergence mem_manager par-mesh-format
-UNIT_TESTS_SUBDIRS = general linalg mesh fem miniapps ceed enzyme dfem
-MINIAPPS_SUBDIRS = dpg/util hooke/operators hooke/preconditioners \
-   hooke/materials hooke/kernels
+UNIT_TESTS_SUBDIRS = general linalg mesh fem miniapps ceed enzyme
+MINIAPPS_SUBDIRS = dpg/util hooke/operators hooke/preconditioners hooke/materials hooke/kernels
 FORMAT_FILES += $(foreach dir,$(TESTS_SUBDIRS),tests/$(dir)/*.?pp)
 FORMAT_FILES += $(foreach dir,$(UNIT_TESTS_SUBDIRS),tests/unit/$(dir)/*.?pp)
 FORMAT_FILES += $(foreach dir,$(MINIAPPS_SUBDIRS),miniapps/$(dir)/*.?pp)
-FORMAT_FILES += config/cmake/config.hpp.in config/config.hpp.in mfem*.hpp
 FORMAT_EXCLUDE = general/tinyxml2.cpp tests/unit/catch.hpp
 FORMAT_LIST = $(filter-out $(FORMAT_EXCLUDE),$(wildcard $(FORMAT_FILES)))
 
@@ -840,29 +833,14 @@ mfem_check_command = \
 # Verify the C++ code styling in MFEM and check that std::cout and std::cerr are
 # not used in the library (use mfem::out and mfem::err instead).
 style:
+	@echo "Applying C++ code style..."
 	@astyle_version="$$($(ASTYLE_BIN) --version)";\
 	 if [ "$$astyle_version" != $(ASTYLE_VER) ]; then\
 	    printf "%s\n" "Invalid astyle version: '$$astyle_version'"\
 	           "Please use: '"$(ASTYLE_VER)"'";\
 	    exit 1;\
 	 fi
-	@err_code=0; \
-	if command -v git 2>&1 > /dev/null && [ -d $(MFEM_DIR)/.git ]; then \
-	   echo "Checking if all git files are selected for formatting ..."; \
-	   ls -1 $(FORMAT_FILES) | sort > format-files-make.txt; \
-	   git -C $(MFEM_DIR) ls-files '*.[ch]pp*' | sort \
-	      > format-files-git.txt; \
-	   cat format-files-make.txt format-files-git.txt | sort | uniq \
-	      > format-files-make-plus-git.txt; \
-	   rm -f format-files-git.txt; \
-	   $(call mfem_check_command,\
-	      diff format-files-make.txt format-files-make-plus-git.txt | \
-	      grep "^> ",\
-	      "All git files are selected for formatting",\
-	      "The above git files are NOT selected for formatting"); \
-	   rm -f format-files-make.txt format-files-make-plus-git.txt; \
-	fi; \
-	echo "Applying C++ code style...";\
+	@err_code=0;\
 	$(call mfem_check_command,\
 	    $(ASTYLE) $(FORMAT_LIST) | grep Formatted,\
 	    "No source files were changed",\

mesh/mesh.cpp

@@ -1616,9 +1616,7 @@ Element::Type Mesh::GetFaceElementType(int Face) const
 
 Array<int> Mesh::GetFaceToBdrElMap() const
 {
-   Array<int> face_to_be(Dim == 1 ? NumOfVertices :
-                         Dim == 2 ? NumOfEdges :
-                         Dim == 3 ? NumOfFaces : 0);
+   Array<int> face_to_be(Dim == 2 ? NumOfEdges : NumOfFaces);
    face_to_be = -1;
    for (int i = 0; i < NumOfBdrElements; i++)
    {
@@ -15236,17 +15234,17 @@ void GeometricFactors::Compute(const GridFunction &nodes,
                         Device::GetDeviceMemoryType();
    if (computed_factors & GeometricFactors::COORDINATES)
    {
-      X.SetSize(vdim*NQ*NE, my_d_mt); // NQ x SDIM x NE
+      X.SetSize(bigint(vdim)*NQ*NE, my_d_mt); // NQ x SDIM x NE
       eval_flags |= QuadratureInterpolator::VALUES;
    }
    if (computed_factors & GeometricFactors::JACOBIANS)
    {
-      J.SetSize(dim*vdim*NQ*NE, my_d_mt); // NQ x SDIM x DIM x NE
+      J.SetSize(bigint(dim)*vdim*NQ*NE, my_d_mt); // NQ x SDIM x DIM x NE
       eval_flags |= QuadratureInterpolator::DERIVATIVES;
    }
    if (computed_factors & GeometricFactors::DETERMINANTS)
    {
-      detJ.SetSize(NQ*NE, my_d_mt); // NQ x NE
+      detJ.SetSize(bigint(NQ)*NE, my_d_mt); // NQ x NE
       eval_flags |= QuadratureInterpolator::DETERMINANTS;
    }
 
@@ -15264,7 +15262,7 @@ void GeometricFactors::Compute(const GridFunction &nodes,
 
    if (elem_restr) // Always true as of 2021-04-27
    {
-      Vector Enodes(vdim*ND*NE, my_d_mt);
+      Vector Enodes(bigint(vdim)*ND*NE, my_d_mt);
       elem_restr->Mult(nodes, Enodes);
       qi->Mult(Enodes, eval_flags, X, J, detJ);
    }
@@ -15306,22 +15304,22 @@ FaceGeometricFactors::FaceGeometricFactors(const Mesh *mesh,
 
    if (flags & FaceGeometricFactors::COORDINATES)
    {
-      X.SetSize(vdim*NQ*NF, my_d_mt);
+      X.SetSize(bigint(vdim)*NQ*NF, my_d_mt);
       eval_flags |= FaceQuadratureInterpolator::VALUES;
    }
    if (flags & FaceGeometricFactors::JACOBIANS)
    {
-      J.SetSize(vdim*(mesh->Dimension() - 1)*NQ*NF, my_d_mt);
+      J.SetSize(bigint(vdim)*(mesh->Dimension() - 1)*NQ*NF, my_d_mt);
       eval_flags |= FaceQuadratureInterpolator::DERIVATIVES;
    }
    if (flags & FaceGeometricFactors::DETERMINANTS)
    {
-      detJ.SetSize(NQ*NF, my_d_mt);
+      detJ.SetSize(bigint(NQ)*NF, my_d_mt);
       eval_flags |= FaceQuadratureInterpolator::DETERMINANTS;
    }
    if (flags & FaceGeometricFactors::NORMALS)
    {
-      normal.SetSize(vdim*NQ*NF, my_d_mt);
+      normal.SetSize(bigint(vdim)*NQ*NF, my_d_mt);
       eval_flags |= FaceQuadratureInterpolator::NORMALS;
    }
 

mesh/mesh.hpp

@@ -3206,22 +3206,10 @@ public:
 
 
 /// Extrude a 1D mesh
-/**
- * @param mesh      1D mesh
- * @param ny        number of transverse elements of the extruded mesh
- * @param sy        physical size in the direction of extrusion
- * @param closed    if false, only the original boundaries are extruded,
- *                  otherwise boundaries are generated all around the domain
- */
 Mesh *Extrude1D(Mesh *mesh, const int ny, const real_t sy,
                 const bool closed = false);
 
 /// Extrude a 2D mesh
-/**
- * @param mesh      2D mesh
- * @param nz        number of transverse elements of the extruded mesh
- * @param sz        physical size in the direction of extrusion
- */
 Mesh *Extrude2D(Mesh *mesh, const int nz, const real_t sz);
 
 /** @brief Constructs the smallest possible [0,1]^dim serial mesh that can be

mesh/mesh_operators.cpp

@@ -63,6 +63,7 @@ ThresholdRefiner::ThresholdRefiner(ErrorEstimator &est)
 
    threshold = 0.0;
    num_marked_elements = 0LL;
+   current_sequence = -1;
 
    non_conforming = -1;
    nc_limit = 0;
@@ -86,6 +87,7 @@ int ThresholdRefiner::MarkWithoutRefining(Mesh & mesh,
    threshold = 0.0;
    num_marked_elements = 0LL;
    refinements.SetSize(0);
+   current_sequence = mesh.GetSequence();
 
    const long long num_elements = mesh.GetGlobalNE();
    if (num_elements >= max_elements) { return STOP; }
@@ -147,6 +149,7 @@ int ThresholdRefiner::ApplyImpl(Mesh &mesh)
 void ThresholdRefiner::Reset()
 {
    estimator.Reset();
+   current_sequence = -1;
    num_marked_elements = 0LL;
    // marked_elements.SetSize(0); // not necessary
 }

mesh/mesh_operators.hpp

@@ -188,6 +188,7 @@ protected:
    long long num_marked_elements;
 
    Array<Refinement> marked_elements;
+   long current_sequence;
 
    int non_conforming;
    int nc_limit;

mesh/mesh_readers.cpp

@@ -1516,15 +1516,12 @@ void Mesh::ReadInlineMesh(std::istream &input, bool generate_edges)
 void Mesh::ReadGmshMesh(std::istream &input, int &curved, int &read_gf)
 {
    string buff;
-   string version;
+   real_t version;
    int binary, dsize;
    input >> version >> binary >> dsize;
-   if (version != "2.2")
+   if (version < 2.2)
    {
-      MFEM_ABORT("Gmsh file version must be 2.2, found version "
-                 << version << ".\n"
-                 "To convert your mesh to the required format, use:\n"
-                 "  gmsh -format msh22 -save -o output.msh input.msh");
+      MFEM_ABORT("Gmsh file version < 2.2");
    }
    if (dsize != sizeof(double))
    {

mesh/pmesh.cpp

@@ -5639,12 +5639,6 @@ Mesh ParMesh::GetSerialMesh(int save_rank) const
       }
    }
 
-   if (MyRank == save_rank)
-   {
-      attribute_sets.Copy(serialmesh.attribute_sets);
-      bdr_attribute_sets.Copy(serialmesh.bdr_attribute_sets);
-   }
-
    MPI_Barrier(MyComm);
    return serialmesh;
 }

mesh/submesh/pncsubmesh.cpp

@@ -78,7 +78,7 @@ ParNCSubMesh::ParNCSubMesh(ParSubMesh& submesh, const ParNCMesh &parent,
 #ifdef MFEM_DEBUG
    // Check all processors have the same number of roots
    {
-      int p[2] = {root_state.Size(), -root_state.Size()};
+      int p[2] = {(int)root_state.Size(), -(int)root_state.Size()};
       MPI_Allreduce(MPI_IN_PLACE, p, 2, MPI_INT, MPI_MIN, submesh.GetComm());
       MFEM_ASSERT(p[0] == -p[1], "Ranks must agree on number of root elements: min "
                   << p[0] << " max " << -p[1] << " local " << root_state.Size() << " MyRank " <<
@@ -154,4 +154,4 @@ ParNCSubMesh::ParNCSubMesh(ParSubMesh& submesh, const ParNCMesh &parent,
 
 } // namespace mfem
 
-#endif // MFEM_USE_MPI
+#endif // MFEM_USE_MPI

mesh/submesh/psubmesh.hpp

@@ -227,29 +227,15 @@ public:
                                            const ParGridFunction &dst);
 
    /**
-   * @brief Check if Mesh @a m is a ParSubMesh.
+   * @brief Check if ParMesh @a m is a ParSubMesh.
    *
-   * @param m The input Mesh
+   * @param m The input ParMesh
    */
-   static bool IsParSubMesh(const Mesh *m)
+   static bool IsParSubMesh(const ParMesh *m)
    {
       return dynamic_cast<const ParSubMesh *>(m) != nullptr;
    }
 
-   /**
-   * @brief Check if Mesh @a sub is a ParSubMesh of Mesh @a parent.
-   *
-   * @param sub The potential submesh Mesh
-   * @param parent The potential parent Mesh
-   */
-   static bool IsParSubMesh(const Mesh* sub, const Mesh* parent)
-   {
-      while (IsParSubMesh(sub) &&
-             (sub = static_cast<const ParSubMesh *>(sub)->GetParent()) &&
-             sub != parent);
-      return sub == parent;
-   }
-
 private:
    ParSubMesh(const ParMesh &parent, SubMesh::From from,
               const Array<int> &attributes);

mesh/submesh/submesh.hpp

@@ -225,20 +225,6 @@ public:
       return dynamic_cast<const SubMesh *>(m) != nullptr;
    }
 
-   /**
-   * @brief Check if Mesh @a sub is a SubMesh of Mesh @a parent.
-   *
-   * @param sub The potential submesh Mesh
-   * @param parent The potential parent Mesh
-   */
-   static bool IsSubMesh(const Mesh* sub, const Mesh* parent)
-   {
-      while (IsSubMesh(sub) &&
-             (sub = static_cast<const SubMesh *>(sub)->GetParent()) &&
-             sub != parent);
-      return sub == parent;
-   }
-
 private:
    /// Private constructor
    SubMesh(const Mesh &parent, From from, const Array<int> &attributes);

miniapps/CMakeLists.txt

@@ -35,7 +35,6 @@ add_subdirectory(multidomain)
 add_subdirectory(nurbs)
 add_subdirectory(parelag)
 add_subdirectory(performance)
-add_subdirectory(plasma)
 add_subdirectory(shifted)
 add_subdirectory(solvers)
 add_subdirectory(spde)

miniapps/electromagnetics/CMakeLists.txt

@@ -43,39 +43,19 @@ endif()
   
   # Add the corresponding tests to the "test" target
   if (MFEM_ENABLE_TESTING)
-    add_test(NAME tesla_1_np=${MFEM_MPI_NP}
+    add_test(NAME tesla_np=4
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
       $<TARGET_FILE:tesla> -no-vis -maxit 2 -cr "0 0 -0.2 0 0 0.2 0.2 0.4 1"
       ${MPIEXEC_POSTFLAGS})
-    add_test(NAME tesla_2_np=${MFEM_MPI_NP}
+
+    add_test(NAME volta_np=4
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:tesla>
-      -no-vis -maxit 2 -m ../../data/inline-hex.mesh -ubbc "0 0 1"
+      $<TARGET_FILE:volta> -no-vis -maxit 2 -dbcs 1 -dbcg -ds "0.0 0.0 0.0 0.2 8.0"
       ${MPIEXEC_POSTFLAGS})
 
-    add_test(NAME volta_1_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:volta>
-      -no-vis -maxit 2 -dbcs 1 -dbcg -ds "0.0 0.0 0.0 0.2 8.0"
-      ${MPIEXEC_POSTFLAGS})
-    add_test(NAME volta_2_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:volta>
-      -no-vis -maxit 2 -m ../../data/square-disc.mesh -dbcs "1 2 3 4 5 6 7 8"
-      -dbcv "0 0 0 0 1 1 1 1"
-      ${MPIEXEC_POSTFLAGS})
-    add_test(NAME volta_3_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:volta>
-      -no-vis -maxit 2 -m ../../data/inline-hex.mesh -dbcs "1 6" -dbcv "0 1"
-      ${MPIEXEC_POSTFLAGS})
-
-    add_test(NAME joule_np=${MFEM_MPI_NP}
+    add_test(NAME joule_np=4
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
       $<TARGET_FILE:joule>
@@ -83,41 +63,12 @@ endif()
       ${MPIEXEC_POSTFLAGS})
 
     if (MFEM_USE_DOUBLE) # otherwise returns MFEM_SKIP_RETURN_VALUE
-      add_test(NAME maxwell_np=${MFEM_MPI_NP}
+      add_test(NAME maxwell_np=4
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
         ${MPIEXEC_PREFLAGS}
         $<TARGET_FILE:maxwell>
         -no-vis -abcs "-1" -dp "-0.3 0.0 0.0 0.3 0.0 0.0 0.1 1 .5 .5"
         ${MPIEXEC_POSTFLAGS})
     endif()
-
-    if (MFEM_USE_GSLIB)
-      add_test(NAME lorentz_1_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:lorentz>
-        -no-vis -er Volta-AMR-Parallel -ec 2 -npt 100 -xmin "0.0 0.0 0.0"
-        -xmax "1.0 1.0 1.0" -pmin "1 0 0" -pmax "1 0 0" -rdf 0 -vt 0 -nt 100
-        ${MPIEXEC_POSTFLAGS})
-      # Setup dependency on volta_3_np=<np>
-      set_tests_properties(volta_3_np=${MFEM_MPI_NP}
-        PROPERTIES FIXTURES_SETUP Volta3)
-      set_tests_properties(lorentz_1_np=${MFEM_MPI_NP}
-        PROPERTIES FIXTURES_REQUIRED Volta3)
-
-      add_test(NAME lorentz_2_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:lorentz>
-        -no-vis -br Tesla-AMR-Parallel -bc 2 -npt 10 -xmin "0.0 0.0 0.0"
-        -xmax "1.0 1.0 1.0" -pmin "0 0.1 0.05" -pmax "0 0.4 0.1" -nt 1000 -rdf 0
-        -vt 0
-        ${MPIEXEC_POSTFLAGS})
-      # Setup dependency on tesla_2_np=<np>
-      set_tests_properties(tesla_2_np=${MFEM_MPI_NP}
-        PROPERTIES FIXTURES_SETUP Tesla2)
-      set_tests_properties(lorentz_2_np=${MFEM_MPI_NP}
-        PROPERTIES FIXTURES_REQUIRED Tesla2)
-    endif()
   endif()
 endif()

miniapps/electromagnetics/makefile

@@ -117,10 +117,10 @@ joule-test-par: joule
 lorentz-test-par: lorentz-test-1 lorentz-test-2
 lorentz-test-1: lorentz volta-test-3
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-er Volta-AMR-Parallel -ec 2 -npt 100 -xmin '0.0 0.0 0.0' -xmax '1.0 1.0 1.0' -pmin '1 0 0' -pmax '1 0 0' -rdf 0 -vt 0 -nt 100)
+	-er Volta-AMR-Parallel -ec 2 -npt 100 -xmin '0.0 0.0 0.0' -xmax '1.0 1.0 1.0' -pmin '1 0 0' -pmax '1 0 0' -rdf 0 -vt 0 -nt 100')
 lorentz-test-2: lorentz tesla-test-2
 	@$(call mfem-test,$<, $(RUN_MPI), Electromagnetic miniapp,\
-	-br Tesla-AMR-Parallel -bc 2 -npt 10 -xmin '0.0 0.0 0.0' -xmax '1.0 1.0 1.0' -pmin '0 0.1 0.05' -pmax '0 0.4 0.1' -nt 1000 -rdf 0 -vt 0)
+	-br Tesla-AMR-Parallel -bc 2 -br Tesla-AMR-Parallel -npt 10 -xmin '0.0 0.0 0.0' -xmax '1.0 1.0 1.0' -pmin '0 0.1 0.05' -pmax '0 0.4 0.1' -nt 1000 -rdf 0 -vt 0)
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk