Merge branch 'master' into Eikonal

.github/workflows/builds-and-tests.yml

@@ -33,7 +33,6 @@ env:
   HYPRE_ARCHIVE: v2.19.0.tar.gz
   HYPRE_TOP_DIR: hypre-2.19.0
   METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
   METIS_TOP_DIR: metis-4.0.3
   MFEM_TOP_DIR: mfem
 
@@ -53,7 +52,6 @@ jobs:
         mpi: [seq, par]
         build-system: [make, cmake]
         hypre-target: [int32]
-        precision: [fp64]
         exclude:
           - os: ubuntu-latest
             build-system: cmake
@@ -77,8 +75,6 @@ jobs:
           - os: ubuntu-latest
             target: dbg
             config-opts: 'CPPFLAGS+=-Og'
-          - os: macos-latest
-            codecov: NO
           - os: windows-latest
             codecov: NO
           - os: windows-latest
@@ -91,7 +87,6 @@ jobs:
             mpi: par
             build-system: cmake
             hypre-target: int32
-            precision: fp64
             # This option can be set to pass additional configuration options to
             # the MFEM configuration command.
             # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
@@ -101,15 +96,7 @@ jobs:
             mpi: par
             build-system: make
             hypre-target: int64
-            precision: fp64
-          - os: ubuntu-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int32
-            precision: fp32
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
 
     runs-on: ${{ matrix.os }}
 
@@ -139,17 +126,6 @@ jobs:
           # Fetch the complete history for codecov to access commits ID
           fetch-depth: 0
 
-      - name: Xcode version setup (MacOS)
-        if: matrix.os == 'macos-latest'
-        run: |
-          XCODE_PATH="/Applications/Xcode_15.3.app"
-          echo "> sudo xcode-select -s ${XCODE_PATH}"
-          sudo xcode-select -s ${XCODE_PATH}
-          echo "> g++ -v"
-          g++ -v
-          echo "> clang++ -v"
-          clang++ -v
-
       # Only get MPI if defined for the job.
       # TODO: It would be nice to have only one step, e.g. with a dedicated
       # action, but I (@adrienbernede) don't see how at the moment.
@@ -193,27 +169,25 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
 
       - name: get hypre
         if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.4
         with:
           archive: ${{ env.HYPRE_ARCHIVE }}
           dir: ${{ env.HYPRE_TOP_DIR }}
           target: ${{ matrix.hypre-target }}
           build-system: make
-          precision: ${{ matrix.precision }}
 
       - name: get hypre (Windows)
         if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.4
         with:
           archive: ${{ env.HYPRE_ARCHIVE }}
           dir: ${{ env.HYPRE_TOP_DIR }}
           target: ${{ matrix.hypre-target }}
           build-system: cmake
-          precision: ${{ matrix.precision }}
 
       # Get Metis through cache, or build it.
       # Install will only run on cache miss.
@@ -223,13 +197,13 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
 
       - name: install metis
         if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
+        uses: mfem/github-actions/build-metis@v2.4
         with:
-          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
+          archive: ${{ env.METIS_ARCHIVE }}
           dir: ${{ env.METIS_TOP_DIR }}
 
       - name: cache vcpkg (Windows)
@@ -254,7 +228,7 @@ jobs:
 
       # MFEM build and test
       - name: build
-        uses: mfem/github-actions/build-mfem@v2.5
+        uses: mfem/github-actions/build-mfem@v2.4
         env:
           VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
         with:
@@ -266,7 +240,6 @@ jobs:
           hypre-dir: ${{ env.HYPRE_TOP_DIR }}
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
-          precision: ${{ matrix.precision }}
           config-options: ${{ matrix.config-opts }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
@@ -309,7 +282,7 @@ jobs:
       # Code coverage (process and upload reports)
       - name: codecov
         if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.5
+        uses: mfem/github-actions/upload-coverage@v2.4
         with:
           name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
           project_dir: ${{ env.MFEM_TOP_DIR }}

.github/workflows/mfem-analysis.yml

@@ -53,11 +53,11 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.5
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
 
       - name: Get Hypre
         if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.5
+        uses: mfem/github-actions/build-hypre@v2.4
         with:
           archive: ${{ env.HYPRE_ARCHIVE }}
           dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +68,18 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
 
       - name: Install Metis
         if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.5
+        uses: mfem/github-actions/build-metis@v2.4
         with:
           archive: ${{ env.METIS_ARCHIVE }}
           dir: ${{ env.METIS_TOP_DIR }}
 
       # MFEM build and test
       - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.5
+        uses: mfem/github-actions/build-mfem@v2.4
         with:
           os: ${{ runner.os }}
           target: opt

.github/workflows/mfem-sanitizer.yml

@@ -44,7 +44,7 @@ jobs:
           path: mfem
 
       - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.5
+        uses: mfem/github-actions/build-mfem@v2.4
         with:
           os: ${{ runner.os }}
           target: opt

.gitignore

@@ -57,8 +57,6 @@ examples/ex2[0-9]
 examples/ex2[0-9]p
 examples/ex3[0-9]
 examples/ex3[0-9]p
-examples/ex4[0-9]
-examples/ex4[0-9]p
 
 examples/refined.mesh
 examples/displaced.mesh
@@ -234,7 +232,7 @@ miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
-miniapps/meshing/mesh-explorer.mesh*
+miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
 miniapps/meshing/mesh-explorer-visit*
 miniapps/meshing/mesh-explorer-paraview/
@@ -371,7 +369,7 @@ miniapps/dpg/ParaView
 miniapps/spde/generate_random_field
 miniapps/spde/ParaView
 
-miniapps/tribol/contact-patch-test
+miniapps/tribol/ContactPatchTest
 
 # Unit test binary and outputs
 tests/unit/output_meshes

.gitlab-ci.yml

@@ -13,9 +13,6 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!
 
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
 
 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use

.gitlab/configs/common.yml

@@ -9,10 +9,6 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
 # We define the following GitLab pipeline variables:
 variables:
 

.gitlab/configs/lassen-config.yml

@@ -35,8 +35,9 @@ variables:
     - when: on_success
 
 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use the pci queue on lassen
-# to speed-up the allocation.
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen
+# to speed-up the allocation.  However this would not be scalable to
+# multiple builds.
 .build_and_test_on_lassen:
   extends: [.on_lassen]
   stage: build_and_test
@@ -44,5 +45,5 @@ variables:
     - echo ${MFEM_DATA_DIR}
     - echo ${SPEC}
     # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
   needs: [setup]

.gitlab/configs/quartz-config.yml

@@ -52,4 +52,4 @@ variables:
     - echo ${JOBID}
     - echo ${MFEM_DATA_DIR}
     - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/lassen-build-and-test.yml

@@ -14,14 +14,14 @@ stages:
   - build_and_test
   - report
 
-opt_mpi_cuda_gcc:
+opt_mpi_cuda_xl_16_1_1_12:
   variables:
-    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
-opt_mpi_cuda_hypre_cuda_gcc:
+opt_mpi_cuda_hypre_cuda_xl:
   variables:
-    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 # Jobs report

.gitlab/scripts/baseline

@@ -32,11 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
   salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
   echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
   exit 1

CHANGELOG

@@ -8,110 +8,77 @@
                                https://mfem.org
 
 
-Version 4.7.1 (development)
+Version 4.6.1 (development)
 ===========================
 
-- Added an MFEM example for the eikonal equation. This new solver is based on
-  the proximal Galerkin method introduced by Keith and Surowiec.
-
-
-Version 4.7, released on May 7, 2024
-====================================
-
-- Added support for single precision (with corresponding hypre build). The MFEM
-  floating point type was generalized from `double` to `real_t`. For details see
-  https://github.com/orgs/mfem/discussions/4207.
-
 Meshing improvements
 --------------------
-- Added the capability to partition (big) serial meshes in serial code, see the
-  new classes MeshPartitioner and MeshPart. This capability is also exposed as a
-  menu option in the mesh-explorer miniapp in miniapps/meshing.
-
-- Added named attribute sets and basic supporting methods to the Mesh class as a
-  convenient means of referring to sets of domain or boundary attribute numbers.
-  See the new Example 39/39p and data/compass.mesh.
-
 - Introduced formulas for refinement of patches in NURBS meshes. Refinement by
   arbitrary integer factors is also enabled, e.g. in the mesh-explorer miniapp.
   NURBS coarsening and knot removal are also introduced.
 
-- Added support for internal boundary elements in nonconforming meshes.
-
-- The ReadCubit Genesis mesh importer has been rewritten to improve readability.
-
 Discretization improvements
 ---------------------------
-- Added a new nonlinear integrator, `HyperbolicFormIntegrator` that implements
+- Introduced support for higher order non conformal Nedelec elements on
+  simplices in ParMesh.
+- Introduced support for internal boundary elements in nonconformal adapted
+  meshes.
+
+- Added functionality for construction of cut-surface and cut-volume
+  IntegrationRules through a moment-fitting approach. The cut is specified by
+  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
+
+- Added a new nonlinear integrator, `HyperbolicFormIntegrator`. This implements 
   both element-wise weak divergence and face-wise numerical flux for a general
-  system of hyperbolic conservation laws. To use the integrator for a specific
+  system of hyperbolic conservation laws. To use this integrator for a specific
   flux function, users can define a derived class of `FluxFunction`. Currently,
-  advection, Burgers, shallow-water and Euler equations (see Example 18/18p) are
+  advection, Burgers', shallow-water, Euler equations (see, Example 18) are
   available.
 
-- Added a capability to construct cut-surface and cut-volume IntegrationRules
-  through a moment-fitting approach. The cut is specified by the zero level set
-  of a Coefficient. See fem/intrules_cut.hpp and the new Example 38.
-
-- Introduced support for high-order nonconforming Nedelec elements on simplices.
-
-GPU computing
--------------
-- Added partial assembly and GPU support for the DG diffusion integrator.
-
-- Efficient GPU-accelerated LOR assembly is now supported on surface meshes.
-
-- Added functionality to automatically configure hypre's compute policy to match
-  MFEM's compute policy when hypre is built with GPU support. Requires version
-  hypre-2.31.0 or later.
-
+GPU support
+----------------------------
 - Added support for full assembly on simplices.
-
-- Added partial assembly for linear elasticity (no sum factorization for now).
-
-- Added functionality for BilinearFormIntegrators to use kernels that work for
-  both tensor and unstructured elements.
-
-- The RAJA backend will use `seq_exec` for serial loop execution when RAJA
-  v2023.06.00 and beyond is detected as `loop_exec` is deprecated.
-
-- API change: The macro MFEM_HYPRE_FORALL (from hypre.hpp) which was intended
-  for internal use, has been removed and replaced by the function template
-  mfem::hypre_forall in general/forall.hpp.
+- Added functionality for BilinearFormIntegrators to use kernels that work for both
+  tensor and unstructured elements.
+- Added partial assembly for linear elasticity. Does not use sum factorization for now.
 
 New and updated examples and miniapps
 -------------------------------------
-- Added a new miniapp illustrating elastic contact based on the Tribol library,
-  (https://github.com/LLNL/Tribol). See miniapps/tribol.
+- Added a new block solver in miniapp/solvers for the Darcy problem.
+  The new solver is based on a Bramble-Pasciak preconditioning. User can
+  use and implement their own preconditioner for the mass matrix.
 
-- Added a miniapp to demonstrate low order refined (LOR) block preconditioning
-  for linear elasticity on GPUs. See miniapps/solvers/lor_elast.
+- Added miniapp to demonstrate new elasticity integrator and unstructured element GPU support,
+  and a block diagonal preconditioner using low order refinement. Allows comparison with
+  currently existing legacy mode integrator. See miniapps/solvers/lor_elast.
 
-- Added a new block solver in miniapp/solvers for the Darcy problem. The new
-  solver is based on a Bramble-Pasciak preconditioning. User can use and
-  implement their own preconditioner for the mass matrix.
-
-- Added a small miniapp for printing the shape functions of a KnotVector. See
-  miniapps/nurbs/nurbs_printfunc.cpp.
-
-- Added two new example codes: 38 and 39/39p described above. Substantially
-  updated Example 18/18p.
+- Added a new mortar contact patch test miniapp using the Tribol interface
+  physics library (see https://github.com/LLNL/Tribol).  See miniapps/tribol.
 
 Miscellaneous
 -------------
+- Added support for single and double precision, with corresponding hypre build.
+  Generalized the floating point type from `double` to `real_t`. For more
+  details see https://github.com/orgs/mfem/discussions/4207.
+
+- The ReadCubit Genesis mesh importer has been rewritten to improve readability.
+
 - Updated the Doxygen documentation style, which now requires Doxygen version
   1.9.8 or later. See the doc/ directory.
 
-- Improved thread safety for global variables in the library, e.g. for IntRules,
-  RefinedIntRules, GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+- Improved thread safety for global variables in the library, for example
+  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
+  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
 
 - PETSc integration now generally requires PETSc version 3.21 or later, though
   depending on the functionality older versions may still work.
 
-- Various other simplifications, extensions, and bugfixes in the code.
-
-- Added GSLIB-based gather-scatter operator.
+- RAJA backend will use seq_exec for serial loop execution when RAJA
+  v2023.06.00 and beyond is detected as loop_exec is deprecated.
 
+- Adding named attribute sets and basic supporting methods to the Mesh class as
+  a convenient means of referring to sets of domain or boundary attribute
+  numbers. Also adding related serial and parallel examples which illustrate.
 
 Version 4.6, released on September 27, 2023
 ===========================================
@@ -132,6 +99,7 @@ Meshing improvements
   * The edge to knot map for NURBS meshes can be determined automatically. It is
     no longer needed to specify this in the NURBS mesh.
   * Added curve interpolation method for NURBS.
+  * Added new small miniapp for printing of shape functions of a KnotVector
   * See miniapps/nurbs for example meshes and miniapps.
 
 Discretization improvements
@@ -178,6 +146,8 @@ Linear and nonlinear solvers
 
 - Added HIP support to the PETSc and SUNDIALS interfaces.
 
+- Efficient GPU-accelerated LOR assembly now supports surface meshes.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new H(div) solver miniapp demonstrating the use of a matrix-free

CMakeLists.txt

@@ -58,7 +58,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.7.1)
+set(${PROJECT_NAME}_VERSION 4.6.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})

INSTALL

@@ -75,8 +75,6 @@ and miniapps. See https://glvis.org and https://mfem.org/building.
 
 Quick start with GNU make
 =========================
-See also: https://mfem.org/building
-
 Serial build:
    make serial -j 4
 
@@ -85,7 +83,6 @@ Parallel build:
    (build METIS 4 in ../metis-4.0 relative to mfem/)
    (build hypre in ../hypre relative to mfem/)
    make parallel -j 4
-(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
 
 CUDA build:
    make cuda -j 4
@@ -119,7 +116,6 @@ Parallel build:
    mkdir <mfem-build-dir> ; cd <mfem-build-dir>
    cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
    make -j 4
-(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
 
 CUDA build:
    (this build requires CMake 3.8 or newer)
@@ -616,13 +612,9 @@ The specific libraries and their options are:
             HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
             HYPRE >= 2.22.1   (HYPRE built with CUDA)
             HYPRE >= 2.23.0   (HYPRE built with HIP)
-            HYPRE >= 2.31.0   (runtime selectable HYPRE execution on CPU/GPU)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
-  MFEM_USE_METIS_5 = YES (default is to use METIS 4). For building instructions,
-  see the following:
-    - METIS 4.0.3: https://mfem.org/building/#parallel-mpi-version-of-mfem
-    - METIS 5.1.0: https://mfem.org/building/#parallel-build-using-metis-5
+  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
   URL: https://github.com/mfem/tpls (MFEM mirror, see above)
   Options: METIS_OPT, METIS_LIB.
   Versions: METIS 4.0.3 or 5.1.0.

config/cmake/modules/FindMUMPS.cmake

@@ -16,22 +16,12 @@
 #   - MUMPS_VERSION
 
 include(MfemCmakeUtilities)
-
-# Toggle which precision of MUMPS to use depending on the precision of MFEM.
-if (MFEM_USE_DOUBLE)
-  set(_mumps_header dmumps_c.h)
-  set(_mumps_lib dmumps)
-elseif(MFEM_USE_SINGLE)
-  set(_mumps_header smumps_c.h)
-  set(_mumps_lib smumps)
-endif()
-
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
-  "include" ${_mumps_header} "lib" ${_mumps_lib}
+  "include" dmumps_c.h "lib" dmumps
   "Paths to headers required by MUMPS."
   "Libraries required by MUMPS."
-  ADD_COMPONENT mumps_common "include" ${_mumps_header} "lib" mumps_common
-  ADD_COMPONENT pord "include" ${_mumps_header} "lib" pord)
+  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
+  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
 
 if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
   try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT

config/config.hpp

@@ -120,15 +120,6 @@ constexpr real_t operator""_r(unsigned long long v)
 
 // Check dependencies:
 
-// Define MFEM_MPI_REAL_T to be the appropriate MPI real type
-#ifdef MFEM_USE_MPI
-#ifdef MFEM_USE_SINGLE
-#define MFEM_MPI_REAL_T MPI_FLOAT
-#elif defined MFEM_USE_DOUBLE
-#define MFEM_MPI_REAL_T MPI_DOUBLE
-#endif
-#endif
-
 // Options that require MPI
 #ifndef MFEM_USE_MPI
 #ifdef MFEM_USE_SUPERLU

config/defaults.mk

@@ -170,18 +170,6 @@ MFEM_USE_PARELAG       = NO
 MFEM_USE_TRIBOL        = NO
 MFEM_USE_ENZYME        = NO
 
-# Process MFEM_PRECISION -> MFEM_USE_SINGLE, MFEM_USE_DOUBLE
-ifneq ($(filter double Double DOUBLE,$(MFEM_PRECISION)),)
-   MFEM_USE_DOUBLE = YES
-   MFEM_USE_SINGLE = NO
-else ifneq ($(filter single Single SINGLE,$(MFEM_PRECISION)),)
-   MFEM_USE_DOUBLE = NO
-   MFEM_USE_SINGLE = YES
-else ifeq ($(MAKECMDGOALS),config)
-   $(error Invalid floating-point precision: \
-     MFEM_PRECISION = $(MFEM_PRECISION))
-endif
-
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
 ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
@@ -331,13 +319,13 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib
+MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib \
+ -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
 ifeq ($(MFEM_USE_SINGLE),YES)
    MUMPS_LIB += -lsmumps
 else
    MUMPS_LIB += -ldmumps
 endif
-MUMPS_LIB += -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
 
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
@@ -388,7 +376,7 @@ GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
 AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt
+AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
 
 # GnuTLS library configuration
 GNUTLS_OPT =

config/makefile

@@ -110,4 +110,4 @@ config-mk:
 
 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out *.dSYM
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out

data/compass.mesh

@@ -92,5 +92,4 @@ vertices
 -0.70710678 -0.70710678
 0 -1
 0.70710678 -0.70710678
-
 mfem_mesh_end

doc/CodeDocumentation.conf.in

@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.7.1
+PROJECT_NUMBER         = v4.6.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/CodeDocumentation.dox

@@ -110,13 +110,9 @@ namespace mfem {
  * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
  * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
  * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
- * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: topology optimization
+ * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
  * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
  * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
- * - <a class="el" href="ex39_8cpp_source.html">Example 39</a>: named mesh attributes
- * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
- * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
- * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -218,8 +214,6 @@ namespace mfem {
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
- * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
- * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs

doc/js/doxygen-awesome-darkmode-toggle.js

@@ -46,7 +46,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
             DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
         })
         // Update the color scheme when the tab is made visible again.
-        // It is possible that the appearance was changed in another tab
+        // It is possible that the appearance was changed in another tab 
         // while this tab was in the background.
         document.addEventListener("visibilitychange", visibilityState => {
             if (document.visibilityState === 'visible') {
@@ -97,7 +97,7 @@ class DoxygenAwesomeDarkModeToggle extends HTMLElement {
      * @returns `true` for dark-mode, `false` for light-mode user preference
      */
     static get userPreference() {
-        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) ||
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
         (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
     }
 

examples/CMakeLists.txt

@@ -45,7 +45,6 @@ list(APPEND ALL_EXE_SRCS
   ex37.cpp
   ex38.cpp
   ex39.cpp
-  ex40.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -88,7 +87,6 @@ if (MFEM_USE_MPI)
     ex36p.cpp
     ex37p.cpp
     ex39p.cpp
-    ex40p.cpp
   )
 endif()
 
@@ -148,10 +146,10 @@ if (MFEM_ENABLE_TESTING)
   # Add CUDA/HIP tests.
   set(DEVICE_EXAMPLES
       # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
+      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
       # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p ex22p ex24p ex25p
-      ex26p ex34p ex35p)
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p
+      ex34p ex35p)
   set(MFEM_TEST_DEVICE)
   if (MFEM_USE_CUDA)
     set(MFEM_TEST_DEVICE "cuda")
@@ -161,11 +159,6 @@ if (MFEM_ENABLE_TESTING)
   if (MFEM_TEST_DEVICE)
     foreach(TEST_NAME ${DEVICE_EXAMPLES})
       set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
-      if (${TEST_NAME} MATCHES "ex14p")
-        list(APPEND THIS_TEST_OPTIONS "-rs" "2" "-rp" "0" "-pa")
-      elseif (${TEST_NAME} MATCHES "ex14")
-        list(APPEND THIS_TEST_OPTIONS "-r" "2" "-pa")
-      endif()
       if (NOT (${TEST_NAME} MATCHES ".*p$"))
         add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
           COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})

examples/ex10p.cpp

@@ -646,7 +646,10 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const
 
 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t energy = 0.5*M.ParInnerProduct(v, v);
+   real_t loc_energy = 0.5*M.InnerProduct(v, v);
+   real_t energy;
+   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type,
+                 MPI_SUM, fespace.GetComm());
    return energy;
 }
 

examples/ex14.cpp

@@ -18,12 +18,6 @@
 //               ex14 -m ../data/amr-quad.mesh -r 3
 //               ex14 -m ../data/amr-hex.mesh
 //               ex14 -m ../data/fichera-amr.mesh
-//               ex14 -pa -r 1 -o 3
-//               ex14 -pa -r 1 -o 3 -m ../data/fichera.mesh
-//
-// Device sample runs:
-//               ex14 -pa -r 2 -d cuda -o 3
-//               ex14 -pa -r 2 -d cuda -o 3 -m ../data/fichera.mesh
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -52,9 +46,7 @@ int main(int argc, char *argv[])
    real_t sigma = -1.0;
    real_t kappa = -1.0;
    real_t eta = 0.0;
-   bool pa = false;
    bool visualization = 1;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -70,13 +62,9 @@ int main(int argc, char *argv[])
                   "One of the three DG penalty parameters, should be positive."
                   " Negative values are replaced with (order+1)^2.");
    args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -89,129 +77,117 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
    //    NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   const int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
 
-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. By default, or if ref_levels < 0,
    //    we choose it to be the largest number that gives a final mesh with no
    //    more than 50,000 elements.
    {
       if (ref_levels < 0)
       {
-         ref_levels = (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
       }
       for (int l = 0; l < ref_levels; l++)
       {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
       }
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
 
-   // 5. Define a finite element space on the mesh. Here we use discontinuous
+   // 4. Define a finite element space on the mesh. Here we use discontinuous
    //    finite elements of the specified order >= 0.
-   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
-   DG_FECollection fec(order, dim, bt);
-   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of unknowns: " << fespace.GetVSize() << endl;
+   FiniteElementCollection *fec = new DG_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of unknowns: " << fespace->GetVSize() << endl;
 
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 5. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system.
-   LinearForm b(&fespace);
+   LinearForm *b = new LinearForm(fespace);
    ConstantCoefficient one(1.0);
    ConstantCoefficient zero(0.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.AddBdrFaceIntegrator(
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->AddBdrFaceIntegrator(
       new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b.Assemble();
+   b->Assemble();
 
-   // 7. Define the solution vector x as a finite element grid function
+   // 6. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero.
-   GridFunction x(&fespace);
+   GridFunction x(fespace);
    x = 0.0;
 
-   // 8. Set up the bilinear form a(.,.) on the finite element space
+   // 7. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    domain integrator and the interior and boundary DG face integrators.
    //    Note that boundary conditions are imposed weakly in the form, so there
    //    is no need for dof elimination. After assembly and finalizing we
    //    extract the corresponding sparse matrix A.
-   BilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
    if (eta > 0)
    {
-      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
-      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
    }
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.Assemble();
-   a.Finalize();
+   a->Assemble();
+   a->Finalize();
+   const SparseMatrix &A = a->SpMat();
 
-   // 9. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+#ifndef MFEM_USE_SUITESPARSE
+   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
    //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
-   //    non-symmetric one. (Note that tolerances are squared: 1e-12 corresponds
-   //    to a relative tolerance of 1e-6).
-   //
-   //    If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   if (pa)
+   //    non-symmetric one.
+   GSSmoother M(A);
+   if (sigma == -1.0)
    {
-      MFEM_VERIFY(sigma == -1.0,
-                  "The case of PA with sigma != -1 is not yet supported.");
-      CG(a, b, x, 1, 500, 1e-12, 0.0);
+      PCG(A, M, *b, x, 1, 500, 1e-12, 0.0);
    }
    else
    {
-      const SparseMatrix &A = a.SpMat();
-#ifndef MFEM_USE_SUITESPARSE
-      GSSmoother M(A);
-      if (sigma == -1.0)
-      {
-         PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
-      }
-      else
-      {
-         GMRES(A, M, b, x, 1, 500, 10, 1e-12, 0.0);
-      }
-#else
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(A);
-      umf_solver.Mult(b, x);
-#endif
+      GMRES(A, M, *b, x, 1, 500, 10, 1e-12, 0.0);
    }
+#else
+   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(*b, x);
+#endif
 
-   // 10. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
+   // 9. Save the refined mesh and the solution. This output can be viewed later
+   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
    ofstream mesh_ofs("refined.mesh");
    mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
    ofstream sol_ofs("sol.gf");
    sol_ofs.precision(8);
    x.Save(sol_ofs);
 
-   // 11. Send the solution by socket to a GLVis server.
+   // 10. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock(vishost, visport);
       sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
    }
 
+   // 11. Free the used memory.
+   delete a;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete mesh;
+
    return 0;
 }

examples/ex14p.cpp

@@ -17,12 +17,6 @@
 //               mpirun -np 4 ex14p -m ../data/inline-segment.mesh -rs 5
 //               mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 3
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -o 3
-//               mpirun -np 4 ex14p -pa -rs 1 -rp 0 -m ../data/fichera.mesh -o 3
-//
-// Device sample runs:
-//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -o 3
-//               mpirun -np 4 ex14p -pa -rs 2 -rp 0 -d cuda -m ../data/fichera.mesh -o 3
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -44,14 +38,11 @@ using namespace mfem;
 
 class CustomSolverMonitor : public IterativeSolverMonitor
 {
-private:
-   const ParMesh &pmesh;
-   ParGridFunction &pgf;
 public:
-   CustomSolverMonitor(const ParMesh &pmesh_,
-                       ParGridFunction &pgf_) :
-      pmesh(pmesh_),
-      pgf(pgf_) {}
+   CustomSolverMonitor(const ParMesh *m,
+                       ParGridFunction *f) :
+      pmesh(m),
+      pgf(f) {}
 
    void MonitorSolution(int i, real_t norm, const Vector &x, bool final)
    {
@@ -59,24 +50,30 @@ public:
       int  visport   = 19916;
       int  num_procs, myid;
 
-      MPI_Comm_size(pmesh.GetComm(), &num_procs);
-      MPI_Comm_rank(pmesh.GetComm(), &myid);
+      MPI_Comm_size(pmesh->GetComm(),&num_procs);
+      MPI_Comm_rank(pmesh->GetComm(),&myid);
 
-      pgf.SetFromTrueDofs(x);
+      pgf->SetFromTrueDofs(x);
 
       socketstream sol_sock(vishost, visport);
       sol_sock << "parallel " << num_procs << " " << myid << "\n";
       sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << pgf
+      sol_sock << "solution\n" << *pmesh << *pgf
                << "window_title 'Iteration no " << i << "'"
                << "keys rRjlc\n" << flush;
    }
+
+private:
+   const ParMesh *pmesh;
+   ParGridFunction *pgf;
 };
 
 int main(int argc, char *argv[])
 {
    // 1. Initialize MPI and HYPRE.
    Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
    Hypre::Init();
 
    // 2. Parse command-line options.
@@ -87,9 +84,7 @@ int main(int argc, char *argv[])
    real_t sigma = -1.0;
    real_t kappa = -1.0;
    real_t eta = 0.0;
-   bool pa = false;
    bool visualization = 1;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -108,17 +103,13 @@ int main(int argc, char *argv[])
                   "One of the three DG penalty parameters, should be positive."
                   " Negative values are replaced with (order+1)^2.");
    args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
-      if (Mpi::Root())
+      if (myid == 0)
       {
          args.PrintUsage(cout);
       }
@@ -128,19 +119,16 @@ int main(int argc, char *argv[])
    {
       kappa = (order+1)*(order+1);
    }
-   if (Mpi::Root())
+   if (myid == 0)
    {
       args.PrintOptions(cout);
    }
 
-   Device device(device_config);
-   if (Mpi::Root()) { device.Print(); }
-
    // 3. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code. NURBS meshes are projected to second order meshes.
-   Mesh mesh(mesh_file);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
 
    // 4. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ser_ref_levels' of uniform refinement. By default,
@@ -149,54 +137,53 @@ int main(int argc, char *argv[])
    {
       if (ser_ref_levels < 0)
       {
-         ser_ref_levels = (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         ser_ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
       }
       for (int l = 0; l < ser_ref_levels; l++)
       {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
       }
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
 
    // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
    {
       for (int l = 0; l < par_ref_levels; l++)
       {
-         pmesh.UniformRefinement();
+         pmesh->UniformRefinement();
       }
    }
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use discontinuous finite elements of the specified order >= 0.
-   const auto bt = pa ? BasisType::GaussLobatto : BasisType::GaussLegendre;
-   DG_FECollection fec(order, dim, bt);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
-   if (Mpi::Root())
+   FiniteElementCollection *fec = new DG_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   if (myid == 0)
    {
       cout << "Number of unknowns: " << size << endl;
    }
 
    // 7. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system.
-   ParLinearForm b(&fespace);
+   ParLinearForm *b = new ParLinearForm(fespace);
    ConstantCoefficient one(1.0);
    ConstantCoefficient zero(0.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.AddBdrFaceIntegrator(
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->AddBdrFaceIntegrator(
       new DGDirichletLFIntegrator(zero, one, sigma, kappa));
-   b.Assemble();
+   b->Assemble();
 
    // 8. Define the solution vector x as a parallel finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero.
-   ParGridFunction x(&fespace);
+   ParGridFunction x(fespace);
    x = 0.0;
 
    // 9. Set up the bilinear form a(.,.) on the finite element space
@@ -205,51 +192,42 @@ int main(int argc, char *argv[])
    //    Note that boundary conditions are imposed weakly in the form, so there
    //    is no need for dof elimination. After serial and parallel assembly we
    //    extract the corresponding parallel matrix A.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
    if (eta > 0)
    {
-      MFEM_VERIFY(!pa, "BR2 not yet compatible with partial assembly.");
-      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
    }
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.Assemble();
-   a.Finalize();
+   a->Assemble();
+   a->Finalize();
 
    // 10. Define the parallel (hypre) matrix and vectors representing a(.,.),
    //     b(.) and the finite element approximation.
-   OperatorHandle A;
+   HypreParMatrix *A = a->ParallelAssemble();
+   HypreParVector *B = b->ParallelAssemble();
+   HypreParVector *X = x.ParallelProject();
 
-   std::unique_ptr<HypreBoomerAMG> amg;
-   if (pa)
-   {
-      A.Reset(&a, false);
-   }
-   else
-   {
-      A.SetType(Operator::Hypre_ParCSR);
-      a.ParallelAssemble(A);
-      amg.reset(new HypreBoomerAMG(*A.As<HypreParMatrix>()));
-   }
+   delete a;
+   delete b;
 
    // 11. Depending on the symmetry of A, define and apply a parallel PCG or
    //     GMRES solver for AX=B using the BoomerAMG preconditioner from hypre.
+   HypreSolver *amg = new HypreBoomerAMG(*A);
    if (sigma == -1.0)
    {
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(500);
-      cg.SetPrintLevel(1);
-      cg.SetOperator(*A);
-      if (amg) { cg.SetPreconditioner(*amg); }
-      cg.Mult(b, x);
+      HyprePCG pcg(*A);
+      pcg.SetTol(1e-12);
+      pcg.SetMaxIter(500);
+      pcg.SetPrintLevel(2);
+      pcg.SetPreconditioner(*amg);
+      pcg.Mult(*B, *X);
    }
    else
    {
-      CustomSolverMonitor monitor(pmesh, x);
+      CustomSolverMonitor monitor(pmesh, &x);
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetAbsTol(0.0);
       gmres.SetRelTol(1e-12);
@@ -257,37 +235,51 @@ int main(int argc, char *argv[])
       gmres.SetKDim(10);
       gmres.SetPrintLevel(1);
       gmres.SetOperator(*A);
-      if (amg) { gmres.SetPreconditioner(*amg); }
+      gmres.SetPreconditioner(*amg);
       gmres.SetMonitor(monitor);
-      gmres.Mult(b, x);
+      gmres.Mult(*B, *X);
    }
+   delete amg;
 
-   // 12. Save the refined mesh and the solution in parallel. This output can
+   // 12. Extract the parallel grid function corresponding to the finite element
+   //     approximation X. This is the local solution on each processor.
+   x = *X;
+
+   // 13. Save the refined mesh and the solution in parallel. This output can
    //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << Mpi::WorldRank();
-      sol_name << "sol." << setfill('0') << setw(6) << Mpi::WorldRank();
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
 
       ofstream mesh_ofs(mesh_name.str().c_str());
       mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
+      pmesh->Print(mesh_ofs);
 
       ofstream sol_ofs(sol_name.str().c_str());
       sol_ofs.precision(8);
       x.Save(sol_ofs);
    }
 
-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << Mpi::WorldSize() << " " << Mpi::WorldRank() << "\n";
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
       sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
+      sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
+   // 15. Free the used memory.
+   delete X;
+   delete B;
+   delete A;
+
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
    return 0;
 }

examples/ex18.hpp

@@ -39,8 +39,8 @@ private:
    // Base Nonlinear Form
    std::unique_ptr<NonlinearForm> nonlinearForm;
    // element-wise inverse mass matrix
-   std::vector<DenseMatrix> invmass; // local scalar inverse mass
-   std::vector<DenseMatrix> weakdiv; // local weak divergence (trial space ByDim)
+   std::vector<DenseMatrix> invmass; // local scalar inverse mass.
+   std::vector<DenseMatrix> weakdiv; // local weakdivergence. Trial space is ByDim.
    // global maximum characteristic speed. Updated by form integrators
    mutable real_t max_char_speed;
    // auxiliary variable used in Mult
@@ -169,9 +169,9 @@ void DGHyperbolicConservationLaws::Mult(const Vector &x, Vector &y) const
 {
    // 0. Reset wavespeed computation before operator application.
    formIntegrator->ResetMaxCharSpeed();
-   // 1. Apply Nonlinear form to obtain an auxiliary result
+   // 1. Apply Nonlinear form to obtain an axiliary result
    //         z = - <F̂(u_h,n), [[v]]>_e
-   //    If weak-divergence is not preassembled, we also have weak-divergence
+   //    If weak-divergencee is not preassembled, we also have weak-divergence
    //         z = - <F̂(u_h,n), [[v]]>_e + (F(u_h), ∇v)
    nonlinearForm->Mult(x, z);
    if (!weakdiv.empty()) // if weak divergence is pre-assembled

examples/ex40.cpp

@@ -1,374 +0,0 @@
-//                                MFEM Example 40
-//
-// Compile with: make ex40
-//
-// Sample runs: ex40 -step 10 -gr 2.0
-//              ex40 -step 10 -gr 2.0 -o 3 -r 1
-//              ex40 -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
-//              ex40 -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
-//
-// Description: This example code demonstrates how to use MFEM to solve the
-//              eikonal equation,
-//
-//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
-//
-//              The solution of this problem coincides with the unique optimum of
-//              the nonlinear program
-//
-//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
-//
-//              which is the foundation for method implemented below.
-//
-//              Following the proximal Galerkin methodology [1] (see also Example
-//              36), we construct a Legendre function for the unit ball
-//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
-//
-//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
-//
-//              although other choices are possible, each leading to a slightly
-//              different algorithm. We then adaptively regularize the optimization
-//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
-//
-//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
-//
-//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
-//
-//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
-//
-//              defined by the nonlinear saddle-point problems
-//
-//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
-//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
-//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
-//
-//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
-//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
-//              method. This example assumes that g = 0 and allows the step size to
-//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
-//
-//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
-//                  preserving finite element method for pointwise bound constraints.
-//                  arXiv:2307.12444 [math.NA]
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-class ZCoefficient : public VectorCoefficient
-{
-protected:
-   GridFunction *psi;
-   real_t alpha;
-
-public:
-   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
-      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
-
-   virtual void Eval(Vector &V, ElementTransformation &T,
-                     const IntegrationPoint &ip);
-   void SetAlpha(real_t alpha_) { alpha = alpha_; }
-};
-
-class DZCoefficient : public MatrixCoefficient
-{
-protected:
-   GridFunction *psi;
-   real_t alpha;
-
-public:
-   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
-      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
-
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
-                     const IntegrationPoint &ip);
-   void SetAlpha(real_t alpha_) { alpha = alpha_; }
-};
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int max_it = 5;
-   int ref_levels = 3;
-   real_t alpha = 1.0;
-   real_t growth_rate = 1.0;
-   real_t newton_scaling = 0.9;
-   real_t tichonov = 1e-1;
-   real_t tol = 1e-4;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&ref_levels, "-r", "--refs",
-                  "Number of h-refinements.");
-   args.AddOption(&max_it, "-mi", "--max-it",
-                  "Maximum number of iterations");
-   args.AddOption(&tol, "-tol", "--tol",
-                  "Stopping criteria based on the difference between"
-                  "successive solution updates");
-   args.AddOption(&alpha, "-step", "--step",
-                  "Initial size alpha");
-   args.AddOption(&growth_rate, "-gr", "--growth-rate",
-                  "Growth rate of the step size alpha");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the mesh file.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-   int sdim = mesh.SpaceDimension();
-
-   MFEM_ASSERT(mesh.bdr_attributes.Size(),
-               "This example does not currently support meshes"
-               " without boundary attributes."
-              )
-
-   // 3. Postprocess the mesh.
-   // 3A. Refine the mesh to increase the resolution.
-   for (int l = 0; l < ref_levels; l++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   // 3B. Interpolate the geometry after refinement to control geometry error.
-   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
-   int curvature_order = max(order,2);
-   mesh.SetCurvature(curvature_order);
-
-   // 4. Define the necessary finite element spaces on the mesh.
-   RT_FECollection RTfec(order, dim);
-   FiniteElementSpace RTfes(&mesh, &RTfec);
-
-   L2_FECollection L2fec(order, dim);
-   FiniteElementSpace L2fes(&mesh, &L2fec);
-
-   cout << "Number of H(div) dofs: "
-        << RTfes.GetTrueVSize() << endl;
-   cout << "Number of L² dofs: "
-        << L2fes.GetTrueVSize() << endl;
-
-   // 5. Define the offsets for the block matrices
-   Array<int> offsets(3);
-   offsets[0] = 0;
-   offsets[1] = RTfes.GetVSize();
-   offsets[2] = L2fes.GetVSize();
-   offsets.PartialSum();
-
-   BlockVector x(offsets), rhs(offsets);
-   x = 0.0; rhs = 0.0;
-
-   // 6. Define the solution vectors as a finite element grid functions
-   //    corresponding to the fespaces.
-   GridFunction u_gf, delta_psi_gf;
-   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
-   u_gf.MakeRef(&L2fes,x,offsets[1]);
-
-   GridFunction psi_old_gf(&RTfes);
-   GridFunction psi_gf(&RTfes);
-   GridFunction u_old_gf(&L2fes);
-
-   // 7. Define initial guesses for the solution variables.
-   delta_psi_gf = 0.0;
-   psi_gf = 0.0;
-   u_gf = 0.0;
-   psi_old_gf = psi_gf;
-   u_old_gf = u_gf;
-
-   // 8. Prepare for glvis output.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-   if (visualization)
-   {
-      sol_sock.open(vishost,visport);
-      sol_sock.precision(8);
-   }
-
-   // 9. Coefficients to be used later.
-   ConstantCoefficient neg_one(-1.0);
-   ConstantCoefficient zero(0.0);
-   ConstantCoefficient tichonov_cf(tichonov);
-   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
-   ZCoefficient Z(sdim, psi_gf, alpha);
-   DZCoefficient DZ(sdim, psi_gf, alpha);
-   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
-   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
-   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
-   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
-
-   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
-   LinearForm b0, b1;
-   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
-   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
-
-   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
-   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
-   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
-
-   BilinearForm a00(&RTfes);
-   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
-   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
-
-   MixedBilinearForm a10(&RTfes,&L2fes);
-   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
-   a10.Assemble();
-   a10.Finalize();
-   SparseMatrix &A10 = a10.SpMat();
-   SparseMatrix *A01 = Transpose(A10);
-
-   BilinearForm a11(&L2fes);
-   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
-   a11.Assemble();
-   a11.Finalize();
-   SparseMatrix &A11 = a11.SpMat();
-
-   // 11. Iterate.
-   int k;
-   int total_iterations = 0;
-   real_t increment_u = 0.1;
-   GridFunction u_tmp(&L2fes);
-   for (k = 0; k < max_it; k++)
-   {
-      u_tmp = u_old_gf;
-      Z.SetAlpha(alpha);
-      DZ.SetAlpha(alpha);
-
-      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
-
-      int j;
-      for ( j = 0; j < 5; j++)
-      {
-         total_iterations++;
-
-         b0.Assemble();
-         b1.Assemble();
-
-         a00.Assemble(false);
-         a00.Finalize(false);
-         SparseMatrix &A00 = a00.SpMat();
-
-         // Construct Schur-complement preconditioner
-         Vector A00_diag(a00.Height());
-         A00.GetDiag(A00_diag);
-         A00_diag.Reciprocal();
-         SparseMatrix *S = Mult_AtDA(*A01, A00_diag);
-
-         BlockDiagonalPreconditioner prec(offsets);
-         prec.SetDiagonalBlock(0,new DSmoother(A00));
-#ifndef MFEM_USE_SUITESPARSE
-         prec.SetDiagonalBlock(1,new GSSmoother(*S));
-#else
-         prec.SetDiagonalBlock(1,new UMFPackSolver(*S));
-#endif
-         prec.owns_blocks = 1;
-
-         BlockOperator A(offsets);
-         A.SetBlock(0,0,&A00);
-         A.SetBlock(1,0,&A10);
-         A.SetBlock(0,1,A01);
-         A.SetBlock(1,1,&A11);
-
-         GMRES(A,prec,rhs,x,0,2000,500,1e-12,0.0);
-         delete S;
-
-         u_tmp -= u_gf;
-         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
-         u_tmp = u_gf;
-
-         // Damped Newton update
-         psi_gf.Add(newton_scaling, delta_psi_gf);
-         a00.Update();
-
-         if (visualization)
-         {
-            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
-                     << flush;
-         }
-
-         mfem::out << "Newton_update_size = " << Newton_update_size << endl;
-
-         if (Newton_update_size < increment_u)
-         {
-            break;
-         }
-      }
-
-      u_tmp = u_gf;
-      u_tmp -= u_old_gf;
-      increment_u = u_tmp.ComputeL2Error(zero);
-
-      mfem::out << "Number of Newton iterations = " << j+1 << endl;
-      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
-
-      u_old_gf = u_gf;
-      psi_old_gf = psi_gf;
-
-      if (increment_u < tol || k == max_it-1)
-      {
-         break;
-      }
-
-      alpha *= max(growth_rate, 1_r);
-
-   }
-
-   mfem::out << "\n Outer iterations: " << k+1
-             << "\n Total iterations: " << total_iterations
-             << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
-             << endl;
-
-   delete A01;
-   return 0;
-}
-
-void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
-                        const IntegrationPoint &ip)
-{
-   MFEM_ASSERT(psi != NULL, "grid function is not set");
-   MFEM_ASSERT(alpha > 0, "alpha is not positive");
-
-   Vector psi_vals(vdim);
-   psi->GetVectorValue(T, ip, psi_vals);
-   real_t norm = psi_vals.Norml2();
-   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
-
-   V = psi_vals;
-   V *= phi;
-}
-
-void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
-                         const IntegrationPoint &ip)
-{
-   MFEM_ASSERT(psi != NULL, "grid function is not set");
-   MFEM_ASSERT(alpha > 0, "alpha is not positive");
-
-   Vector psi_vals(height);
-   psi->GetVectorValue(T, ip, psi_vals);
-   real_t norm = psi_vals.Norml2();
-   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
-
-   K = 0.0;
-   for (int i = 0; i < height; i++)
-   {
-      K(i,i) = phi;
-      for (int j = 0; j < height; j++)
-      {
-         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
-      }
-   }
-}

examples/ex40p.cpp

@@ -1,436 +0,0 @@
-//                       MFEM Example 40 - Parallel Version
-//
-// Compile with: make ex40p
-//
-// Sample runs: mpirun -np 4 ex40p -step 10 -gr 2.0
-//              mpirun -np 4 ex40p -step 10 -gr 2.0 -o 3 -r 1
-//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 4 -m ../data/l-shape.mesh
-//              mpirun -np 4 ex40p -step 10 -gr 2.0 -r 2 -m ../data/fichera.mesh
-//
-// Description: This example code demonstrates how to use MFEM to solve the
-//              eikonal equation,
-//
-//                      |∇𝑢| = 1 in Ω,  𝑢 = g on ∂Ω.
-//
-//              The solution of this problem coincides with the unique optimum of
-//              the nonlinear program
-//
-//                   maximize ∫_Ω 𝑢 d𝑥 subject to |∇𝑢| ≤ 1,   𝑢 = g on Ω,      (⋆)
-//
-//              which is the foundation for method implemented below.
-//
-//              Following the proximal Galerkin methodology [1] (see also Example
-//              36), we construct a Legendre function for the unit ball
-//              𝐵₁ := {𝑥 ∈ Rⁿ | |𝑥| < 1}. Our choice is the Hellinger entropy,
-//
-//                    h(𝑥) = −( 1 − |𝑥|² )^{1/2},
-//
-//              although other choices are possible, each leading to a slightly
-//              different algorithm. We then adaptively regularize the optimization
-//              problem (⋆) with the Bregman divergence of the Hellinger entropy,
-//
-//                 maximize  ∫_Ω 𝑢 d𝑥 - αₖ⁻¹ Dₕ(∇𝑢,∇𝑢ₖ₋₁)  subject to  𝑢 = g on Ω.
-//
-//              This results in a sequence of functions ( 𝜓ₖ , 𝑢ₖ ),
-//
-//                      𝑢ₖ → 𝑢,    𝜓ₖ/|𝜓ₖ| → ∇𝑢    as k → \infty,
-//
-//              defined by the nonlinear saddle-point problems
-//
-//               Find 𝜓ₖ ∈ H(div,Ω) and 𝑢ₖ ∈ L²(Ω) such that
-//               ( Zₖ(𝜓ₖ) , τ ) + ( 𝑢ₖ , ∇⋅τ ) = ⟨ g , τ⋅n ⟩         ∀ τ ∈ H(div,Ω)
-//               ( ∇⋅𝜓ₖ , v )                 = ( ∇⋅𝜓ₖ₋₁ - 1 , v )   ∀ v ∈ L²(Ω)
-//
-//              where Zₖ(𝜓) := ∇h⁻¹(αₖ 𝜓) = 𝜓 / ( αₖ⁻² + |𝜓|² )^{1/2} and step size
-//              αₖ > 0. These saddle-point problems are solved using a damped Newton's
-//              method. This example assumes that g = 0 and allows the step size to
-//              grow geometrically, αₖ = α₀rᵏ, where r ≥ 1 is the growth rate.
-//
-//              [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
-//                  preserving finite element method for pointwise bound constraints.
-//                  arXiv:2307.12444 [math.NA]
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-class ZCoefficient : public VectorCoefficient
-{
-protected:
-   ParGridFunction *psi;
-   real_t alpha;
-
-public:
-   ZCoefficient(int vdim, ParGridFunction &psi_, real_t alpha_ = 1.0)
-      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
-
-   virtual void Eval(Vector &V, ElementTransformation &T,
-                     const IntegrationPoint &ip);
-   void SetAlpha(real_t alpha_) { alpha = alpha_; }
-};
-
-class DZCoefficient : public MatrixCoefficient
-{
-protected:
-   ParGridFunction *psi;
-   real_t alpha;
-
-public:
-   DZCoefficient(int height, ParGridFunction &psi_, real_t alpha_ = 1.0)
-      : MatrixCoefficient(height),  psi(&psi_), alpha(alpha_) { }
-
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
-                     const IntegrationPoint &ip);
-   void SetAlpha(real_t alpha_) { alpha = alpha_; }
-};
-
-int main(int argc, char *argv[])
-{
-   // 0. Initialize MPI and HYPRE.
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int max_it = 5;
-   int ref_levels = 3;
-   real_t alpha = 1.0;
-   real_t growth_rate = 1.0;
-   real_t newton_scaling = 0.9;
-   real_t tichonov = 1e-1;
-   real_t tol = 1e-4;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&ref_levels, "-r", "--refs",
-                  "Number of h-refinements.");
-   args.AddOption(&max_it, "-mi", "--max-it",
-                  "Maximum number of iterations");
-   args.AddOption(&tol, "-tol", "--tol",
-                  "Stopping criteria based on the difference between"
-                  "successive solution updates");
-   args.AddOption(&alpha, "-step", "--step",
-                  "Initial size alpha");
-   args.AddOption(&growth_rate, "-gr", "--growth-rate",
-                  "Growth rate of the step size alpha");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 2. Read the mesh from the mesh file.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-   int sdim = mesh.SpaceDimension();
-
-   MFEM_ASSERT(mesh.bdr_attributes.Size(),
-               "This example does not currently support meshes"
-               " without boundary attributes."
-              )
-
-   // 3. Postprocess the mesh.
-   // 3A. Refine the mesh to increase the resolution.
-   for (int l = 0; l < ref_levels; l++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   // 3B. Interpolate the geometry after refinement to control geometry error.
-   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
-   int curvature_order = max(order,2);
-   mesh.SetCurvature(curvature_order);
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // 4. Define the necessary finite element spaces on the mesh.
-   RT_FECollection RTfec(order, dim);
-   ParFiniteElementSpace RTfes(&pmesh, &RTfec);
-
-   L2_FECollection L2fec(order, dim);
-   ParFiniteElementSpace L2fes(&pmesh, &L2fec);
-
-   int num_dofs_RT = RTfes.GlobalTrueVSize();
-   int num_dofs_L2 = L2fes.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of H(div) dofs: "
-           << num_dofs_RT << endl;
-      cout << "Number of L² dofs: "
-           << num_dofs_L2 << endl;
-   }
-
-   // 5. Define the offsets for the block matrices
-   Array<int> offsets(3);
-   offsets[0] = 0;
-   offsets[1] = RTfes.GetVSize();
-   offsets[2] = L2fes.GetVSize();
-   offsets.PartialSum();
-
-   Array<int> toffsets(3);
-   toffsets[0] = 0;
-   toffsets[1] = RTfes.GetTrueVSize();
-   toffsets[2] = L2fes.GetTrueVSize();
-   toffsets.PartialSum();
-
-   BlockVector x(offsets), rhs(offsets);
-   x = 0.0; rhs = 0.0;
-
-   BlockVector tx(toffsets), trhs(toffsets);
-   tx = 0.0; trhs = 0.0;
-
-   // 6. Define the solution vectors as a finite element grid functions
-   //    corresponding to the fespaces.
-   ParGridFunction u_gf, delta_psi_gf;
-   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
-   u_gf.MakeRef(&L2fes,x,offsets[1]);
-
-   ParGridFunction psi_old_gf(&RTfes);
-   ParGridFunction psi_gf(&RTfes);
-   ParGridFunction u_old_gf(&L2fes);
-
-   // 7. Define initial guesses for the solution variables.
-   delta_psi_gf = 0.0;
-   psi_gf = 0.0;
-   u_gf = 0.0;
-   psi_old_gf = psi_gf;
-   u_old_gf = u_gf;
-
-   // 8. Prepare for glvis output.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-   if (visualization)
-   {
-      sol_sock.open(vishost,visport);
-      sol_sock.precision(8);
-   }
-
-   // 9. Coefficients to be used later.
-   ConstantCoefficient neg_one(-1.0);
-   ConstantCoefficient zero(0.0);
-   ConstantCoefficient tichonov_cf(tichonov);
-   ConstantCoefficient neg_tichonov_cf(-1.0*tichonov);
-   ZCoefficient Z(sdim, psi_gf, alpha);
-   DZCoefficient DZ(sdim, psi_gf, alpha);
-   ScalarVectorProductCoefficient neg_Z(-1.0, Z);
-   DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
-   DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
-   SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
-
-   // 10. Assemble constant matrices/vectors to avoid reassembly in the loop.
-   ParLinearForm b0, b1;
-   b0.MakeRef(&RTfes,rhs.GetBlock(0),0);
-   b1.MakeRef(&L2fes,rhs.GetBlock(1),0);
-
-   b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
-   b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
-   b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
-
-   ParBilinearForm a00(&RTfes);
-   a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
-   a00.AddDomainIntegrator(new VectorFEMassIntegrator(tichonov_cf));
-
-   ParMixedBilinearForm a10(&RTfes,&L2fes);
-   a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
-   a10.Assemble();
-   a10.Finalize();
-   HypreParMatrix *A10 = a10.ParallelAssemble();
-
-   HypreParMatrix *A01 = A10->Transpose();
-
-   ParBilinearForm a11(&L2fes);
-   a11.AddDomainIntegrator(new MassIntegrator(neg_tichonov_cf));
-   a11.Assemble();
-   a11.Finalize();
-   HypreParMatrix *A11 = a11.ParallelAssemble();
-
-   // 11. Iterate.
-   int k;
-   int total_iterations = 0;
-   real_t increment_u = 0.1;
-   ParGridFunction u_tmp(&L2fes);
-   for (k = 0; k < max_it; k++)
-   {
-      u_tmp = u_old_gf;
-      Z.SetAlpha(alpha);
-      DZ.SetAlpha(alpha);
-
-      if (myid == 0)
-      {
-         mfem::out << "\nOUTER ITERATION " << k+1 << endl;
-      }
-
-      int j;
-      for ( j = 0; j < 5; j++)
-      {
-         total_iterations++;
-
-         b0.Assemble();
-         b0.ParallelAssemble(trhs.GetBlock(0));
-
-         b1.Assemble();
-         b1.ParallelAssemble(trhs.GetBlock(1));
-
-         a00.Assemble(false);
-         a00.Finalize(false);
-         HypreParMatrix *A00 = a00.ParallelAssemble();
-
-         // Construct Schur-complement preconditioner
-         HypreParVector A00_diag(MPI_COMM_WORLD, A00->GetGlobalNumRows(),
-                                 A00->GetRowStarts());
-         A00->GetDiag(A00_diag);
-         HypreParMatrix S_tmp(*A01);
-         S_tmp.InvScaleRows(A00_diag);
-         HypreParMatrix *S = ParMult(A10, &S_tmp, true);
-
-         BlockDiagonalPreconditioner prec(toffsets);
-         HypreBoomerAMG P00(*A00);
-         P00.SetPrintLevel(0);
-         HypreBoomerAMG P11(*S);
-         P11.SetPrintLevel(0);
-         prec.SetDiagonalBlock(0,&P00);
-         prec.SetDiagonalBlock(1,&P11);
-
-         BlockOperator A(toffsets);
-         A.SetBlock(0,0,A00);
-         A.SetBlock(1,0,A10);
-         A.SetBlock(0,1,A01);
-         A.SetBlock(1,1,A11);
-
-         GMRESSolver gmres(MPI_COMM_WORLD);
-         gmres.SetPrintLevel(-1);
-         gmres.SetRelTol(1e-8);
-         gmres.SetMaxIter(2000);
-         gmres.SetKDim(500);
-         gmres.SetOperator(A);
-         gmres.SetPreconditioner(prec);
-         gmres.Mult(trhs,tx);
-         delete S;
-         delete A00;
-
-         delta_psi_gf.SetFromTrueDofs(tx.GetBlock(0));
-         u_gf.SetFromTrueDofs(tx.GetBlock(1));
-
-         u_tmp -= u_gf;
-         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
-         u_tmp = u_gf;
-
-         // Damped Newton update
-         psi_gf.Add(newton_scaling, delta_psi_gf);
-         a00.Update();
-
-         if (visualization)
-         {
-            sol_sock << "parallel " << num_procs << " " << myid << "\n";
-            sol_sock << "solution\n" << pmesh << u_gf << "window_title 'Discrete solution'"
-                     << flush;
-         }
-
-         if (myid == 0)
-         {
-            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
-         }
-
-         if (Newton_update_size < increment_u)
-         {
-            break;
-         }
-      }
-
-      u_tmp = u_gf;
-      u_tmp -= u_old_gf;
-      increment_u = u_tmp.ComputeL2Error(zero);
-
-      if (myid == 0)
-      {
-         mfem::out << "Number of Newton iterations = " << j+1 << endl;
-         mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
-      }
-
-      u_old_gf = u_gf;
-      psi_old_gf = psi_gf;
-
-      if (increment_u < tol || k == max_it-1)
-      {
-         break;
-      }
-
-      alpha *= max(growth_rate, 1_r);
-
-   }
-
-   // 12. Print stats.
-   if (myid == 0)
-   {
-      mfem::out << "\n Outer iterations: " << k+1
-                << "\n Total iterations: " << total_iterations
-                << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
-                << endl;
-   }
-
-   // 13. Free the used memory.
-   delete A01;
-   delete A10;
-   delete A11;
-   return 0;
-}
-
-void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
-                        const IntegrationPoint &ip)
-{
-   MFEM_ASSERT(psi != NULL, "grid function is not set");
-   MFEM_ASSERT(alpha > 0, "alpha is not positive");
-
-   Vector psi_vals(vdim);
-   psi->GetVectorValue(T, ip, psi_vals);
-   real_t norm = psi_vals.Norml2();
-   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
-
-   V = psi_vals;
-   V *= phi;
-}
-
-void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
-                         const IntegrationPoint &ip)
-{
-   MFEM_ASSERT(psi != NULL, "grid function is not set");
-   MFEM_ASSERT(alpha > 0, "alpha is not positive");
-
-   Vector psi_vals(height);
-   psi->GetVectorValue(T, ip, psi_vals);
-   real_t norm = psi_vals.Norml2();
-   real_t phi = 1.0 / sqrt(1.0/(alpha*alpha) + norm*norm);
-
-   K = 0.0;
-   for (int i = 0; i < height; i++)
-   {
-      K(i,i) = phi;
-      for (int j = 0; j < height; j++)
-      {
-         K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
-      }
-   }
-}

examples/ex41.cpp

@@ -0,0 +1,384 @@
+//                                MFEM Example 41
+//
+// Compile with: make ex41
+//
+// Sample runs: ex41 -o 2
+//              ex41 -o 1 -r 4
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              Eikonal equation,
+//
+//                      |∇u| = 1 in Ω,  u = g on ∂Ω.
+//
+//              This example constructs a fast converging sequence,
+//
+//                      uₖ → u  as k → \infty,
+//
+//              by using in Newton's method to solve the sequence of nonlinear
+//              saddle-point problems
+//
+//               Find ψₖ ∈ H(div,Ω) and uₖ ∈ L²(Ω) such that
+//               ( Zₖ(ψₖ) , τ ) + ( uₖ , ∇⋅τ ) = 0                    ∀ τ ∈ H(div,Ω)
+//               ( ∇⋅ψₖ , v )                 = ( -1 + ∇⋅ψₖ₋₁ , v )   ∀ v ∈ L²(Ω)
+//
+//              where Zₖ(ψ) = ψ / ( 1/αₖ + |ψ|² )^{1/2} and αₖ > 0.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height, true),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+  //  MFEM_ASSERT(mesh.bdr_attributes.Size(),
+  //   "This example does not currently support meshes"
+  //   " without boundary attributes."
+  //  )
+
+   bool zero_average = not mesh.bdr_attributes.Size();
+   if (zero_average)
+   {
+      cout << "\nThe domain has no boundary. "
+           << "Solving for zero-average solution.\n" << endl;
+   }
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   RT_FECollection RTfec(order, dim);
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   L2_FECollection L2fec(order, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec);
+
+   cout << "Number of Hdiv finite element unknowns: "
+        << RTfes.GetTrueVSize() << endl;
+   cout << "Number of L2 finite element unknowns: "
+        << L2fes.GetTrueVSize() << endl;
+
+   // 5. Determine the list of true (i.e., conforming) essential boundary dofs.
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = RTfes.GetVSize();
+   offsets[2] = L2fes.GetVSize();
+   if (zero_average)
+   {
+      offsets.Append(1);
+   }
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 6. Define an initial guess for the solution.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+
+   // 7. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_psi_gf;
+
+   delta_psi_gf.MakeRef(&RTfes,x,offsets[0]);
+   u_gf.MakeRef(&L2fes,x,offsets[1]);
+   delta_psi_gf = 0.0;
+
+   GridFunction psi_old_gf(&RTfes);
+   GridFunction psi_gf(&RTfes);
+   GridFunction u_old_gf(&L2fes);
+   u_old_gf = 0.0;
+
+   // 8. Define the function coefficients for the solution and use them to
+   //    initialize the initial guess
+   psi_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   SparseMatrix A21(1, L2fes.GetVSize());
+   if (zero_average)
+   {
+      Array<int> rows(L2fes.GetVSize());
+      Vector values(L2fes.GetVSize());
+      for (int i = 0; i < L2fes.GetVSize(); i++)
+      {
+        rows[i] = i;
+      }
+      values = 1.0;
+      A21.AddRow(0,rows,values);
+   }
+   A21.Finalize();
+   SparseMatrix *A12 = Transpose(A21);
+
+   // 10. Iterate
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   for (k = 0; k < max_it; k++)
+   {
+      GridFunction u_tmp(&L2fes);
+      u_tmp = u_old_gf;
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         ConstantCoefficient alpha_cf(alpha);
+
+         LinearForm b0,b1;
+         b0.Update(&RTfes,rhs.GetBlock(0),0);
+         b1.Update(&L2fes,rhs.GetBlock(1),0);
+
+         ZCoefficient Z(sdim, psi_gf, alpha);
+         DZCoefficient DZ(sdim, psi_gf, alpha);
+
+         ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+         b0.AddDomainIntegrator(new VectorFEDomainLFIntegrator(neg_Z));
+         b0.Assemble();
+
+         DivergenceGridFunctionCoefficient div_psi_cf(&psi_gf);
+         DivergenceGridFunctionCoefficient div_psi_old_cf(&psi_old_gf);
+         SumCoefficient psi_old_minus_psi(div_psi_old_cf, div_psi_cf, 1.0, -1.0);
+         b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+         b1.AddDomainIntegrator(new DomainLFIntegrator(psi_old_minus_psi));
+         b1.Assemble();
+
+         BilinearForm a00(&RTfes);
+         a00.AddDomainIntegrator(new VectorFEMassIntegrator(DZ));
+         ConstantCoefficient eps(1e-1);
+         a00.AddDomainIntegrator(new VectorFEMassIntegrator(eps));
+         a00.Assemble();
+         a00.Finalize();
+         SparseMatrix &A00 = a00.SpMat();
+
+         MixedBilinearForm a10(&RTfes,&L2fes);
+         a10.AddDomainIntegrator(new VectorFEDivergenceIntegrator());
+         a10.Assemble();
+        //  Array<int> dof_elim_array(1);
+        //  int dof_elim = 10;
+        //  dof_elim_array[0] = dof_elim;
+         a10.Finalize();
+         SparseMatrix &A10 = a10.SpMat();
+        //  A10.EliminateRow(dof_elim);
+
+         SparseMatrix *A01 = Transpose(A10);
+
+         BilinearForm a11(&L2fes);
+         ConstantCoefficient neg_eps(-1e-2);
+         a11.AddDomainIntegrator(new MassIntegrator(neg_eps));
+         a11.Assemble(false);
+         a11.Finalize();
+         SparseMatrix &A11 = a11.SpMat();
+
+        //  BilinearForm a11(&L2fes);
+        //  a11.AddDomainIntegrator(new MassIntegrator(zero));
+        //  a11.Assemble(false);
+        //  SparseMatrix A11;
+        //  a11.FormSystemMatrix(dof_elim_array, A11);
+        //  A11.Set(dof_elim, dof_elim, 1.0);
+        //  rhs[offsets[1] + dof_elim] = 0.0;
+
+        //  BlockOperator A(offsets);
+        //  A.SetBlock(0,0,&A00);
+        //  A.SetBlock(1,0,&A10);
+        //  A.SetBlock(0,1,A01);
+
+        //  BlockDiagonalPreconditioner prec(offsets);
+        //  prec.SetDiagonalBlock(0,new GSSmoother(A00));
+        //  prec.SetDiagonalBlock(1,new GSSmoother(A11));
+        //  prec.owns_blocks = 1;
+
+        //  GMRES(A,prec,rhs,x,0,10000,500,1e-12,0.0);
+
+         BlockMatrix A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,&A10);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,1,&A11);
+         if (zero_average)
+         {
+            A.SetBlock(1,2,A12);
+            A.SetBlock(2,1,&A21);
+         }
+
+         SparseMatrix * A_mono = A.CreateMonolithic();
+         UMFPackSolver umf(*A_mono);
+         umf.Mult(rhs,x);
+
+         delta_psi_gf.MakeRef(&RTfes, x.GetBlock(0), 0);
+         u_gf.MakeRef(&L2fes, x.GetBlock(1), 0);
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         real_t gamma = 0.9;
+         delta_psi_gf *= gamma;
+         psi_gf += delta_psi_gf;
+
+         if (visualization)
+         {
+            // sol_sock << "solution\n" << mesh << psi_gf << "window_title 'Discrete solution'"
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         delete A01;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      // alpha *= 2.0;
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << RTfes.GetTrueVSize() + L2fes.GetTrueVSize()
+             << endl;
+
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/alpha + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                                                const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/alpha + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+    K(i,i) = phi;
+    for (int j = 0; j < height; j++)
+      {
+        K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}

examples/ex41b.cpp

@@ -0,0 +1,341 @@
+//                                MFEM Example 41
+//
+// Compile with: make ex41
+//
+// Sample runs: ex41 -o 2
+//              ex41 -o 2 -r 4
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              Eikonal equation,
+//
+//                      |∇u| = 1 in Ω,  u = g on ∂Ω.
+//
+//              This example constructs a fast converging sequence,
+//
+//                      uₖ → u  as k → \infty,
+//
+//              by using in Newton's method to solve the sequence of nonlinear
+//              saddle-point problems
+//
+//               Find qₖ ∈ H¹₀(Ω) and uₖ ∈ H¹₀(Ω) such that
+//               ( ϕₖ(|∇qₖ|) ∇qₖ , ∇w ) + ( ∇uₖ , ∇w ) = 0           ∀ w ∈ H¹₀(Ω)
+//               ( ∇qₖ , ∇v )        = ( -1 , v ) + ( ∇qₖ₋₁ , ∇v )   ∀ v ∈ H¹₀(Ω)
+//
+//              where ϕₖ(s) = 1 / ( 1/αₖ + s² )^{1/2} and αₖ > 0.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   GridFunction *q;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, GridFunction &q_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), q(&q_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   GridFunction *q;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, GridFunction &q_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height, true),  q(&q_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Step size alpha");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   H1_FECollection H1fec(order, dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   cout << "Number of dofs: "
+        << H1fes.GetTrueVSize() * 2 << endl;
+
+   // 5. Determine the list of true (i.e., conforming) essential boundary dofs.
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+   else
+   {
+    ess_tdof_list.Append(0);
+   }
+   
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = H1fes.GetVSize();
+   offsets[2] = H1fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 6. Define an initial guess for the solution.
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+
+   // 7. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_q_gf;
+
+   delta_q_gf.MakeRef(&H1fes,x,offsets[0]);
+   u_gf.MakeRef(&H1fes,x,offsets[1]);
+   delta_q_gf = 0.0;
+
+   GridFunction q_old_gf(&H1fes);
+   GridFunction q_gf(&H1fes);
+   GridFunction u_old_gf(&H1fes);
+   q_old_gf = 0.0;
+   u_old_gf = 0.0;
+
+   // 8. Define the function coefficients for the solution and use them to
+   //    initialize the initial guess
+   q_gf = 0.0;
+   q_old_gf = q_gf;
+   u_old_gf = u_gf;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 10. Iterate
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   for (k = 0; k < max_it; k++)
+   {
+      GridFunction u_tmp(&H1fes);
+      u_tmp = u_old_gf;
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      ConstantCoefficient alpha_cf(alpha);
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         LinearForm b0,b1;
+         b0.Update(&H1fes,rhs.GetBlock(0),0);
+         b1.Update(&H1fes,rhs.GetBlock(1),0);
+
+         ZCoefficient Z(sdim, q_gf, alpha);
+         DZCoefficient DZ(sdim, q_gf, alpha);
+
+         ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+         b0.AddDomainIntegrator(new DomainLFGradIntegrator(neg_Z));
+         b0.Assemble();
+
+         GradientGridFunctionCoefficient grad_q_cf(&q_gf);
+         GradientGridFunctionCoefficient grad_q_old_cf(&q_old_gf);
+         VectorSumCoefficient grad_q_old_minus_q(grad_q_old_cf, grad_q_cf, 1.0, -1.0);
+         b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+         b1.AddDomainIntegrator(new DomainLFGradIntegrator(grad_q_old_minus_q));
+         b1.Assemble();
+
+         BilinearForm a00(&H1fes);
+        //  a00.AddDomainIntegrator(new DiffusionIntegrator());
+         a00.AddDomainIntegrator(new DiffusionIntegrator(DZ));
+         a00.Assemble();
+         a00.EliminateVDofs(ess_tdof_list, mfem::Operator::DIAG_ZERO);
+        //  a00.EliminateVDofs(ess_tdof_list,x.GetBlock(0),rhs.GetBlock(0),
+        //                           mfem::Operator::DIAG_ONE);
+         a00.Finalize();
+         SparseMatrix &A00 = a00.SpMat();
+
+         BilinearForm a10(&H1fes);
+         a10.AddDomainIntegrator(new DiffusionIntegrator());
+         a10.Assemble();
+         a10.EliminateVDofs(ess_tdof_list,x.GetBlock(0),rhs.GetBlock(1),
+                                  mfem::Operator::DIAG_ONE);
+         a10.Finalize();
+         SparseMatrix &A10 = a10.SpMat();
+
+         SparseMatrix *A01 = Transpose(A10);
+
+        //  BlockOperator A(offsets);
+        //  A.SetBlock(0,0,&A00);
+        //  A.SetBlock(1,0,&A10);
+        //  A.SetBlock(0,1,A01);
+
+        //  BlockDiagonalPreconditioner prec(offsets);
+        //  prec.SetDiagonalBlock(0,new GSSmoother(A00));
+        //  prec.SetDiagonalBlock(1,new GSSmoother(A11));
+        //  prec.owns_blocks = 1;
+
+        //  GMRES(A,prec,rhs,x,0,10000,500,1e-12,0.0);
+
+         BlockMatrix A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(0,1,A01);
+         A.SetBlock(1,0,&A10);
+
+         SparseMatrix * A_mono = A.CreateMonolithic();
+         UMFPackSolver umf(*A_mono);
+         umf.Mult(rhs,x);
+
+         delta_q_gf.MakeRef(&H1fes, x.GetBlock(0), 0);
+         u_gf.MakeRef(&H1fes, x.GetBlock(1), 0);
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         real_t gamma = 1.0;
+         delta_q_gf *= gamma;
+         q_gf += delta_q_gf;
+
+         if (visualization)
+         {
+            sol_sock << "solution\n" << mesh << u_tmp << "window_title 'Discrete solution'"
+            // sol_sock << "solution\n" << mesh << q_gf << "window_title 'Discrete solution'"
+            // sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         delete A01;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      q_old_gf = q_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      alpha *= 2.0;
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << H1fes.GetTrueVSize() * 2
+             << endl;
+
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(q != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector gradq(vdim);
+   q->GetGradient(T,gradq);
+   real_t norm = gradq.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/alpha + norm*norm);
+
+   V = gradq;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                                                const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(q != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector gradq(height);
+   q->GetGradient(T,gradq);
+   real_t norm = gradq.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/alpha + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+    K(i,i) = phi;
+    for (int j = 0; j < height; j++)
+      {
+        K(i,j) -= gradq(i) * gradq(j) * pow(phi, 3);
+      }
+   }
+}

examples/ex41c.cpp

@@ -0,0 +1,353 @@
+//                                MFEM Example 41
+//
+// Compile with: make ex41
+//
+// Sample runs: ex41 -o 2
+//              ex41 -o 1 -r 4
+//
+// Description: This example code demonstrates how to use MFEM to solve the
+//              Eikonal equation,
+//
+//                      |∇u| = 1 in Ω,  u = g on ∂Ω.
+//
+//              This example constructs a fast converging sequence,
+//
+//                      uₖ → u  as k → \infty,
+//
+//              by using in Newton's method to solve the sequence of nonlinear
+//              saddle-point problems
+//
+//               Find ψₖ ∈ L²(Ω)ⁿ and uₖ ∈ H¹₀(Ω) such that
+//               ( Zₖ(ψₖ) , τ ) + ( ∇uₖ , τ ) = 0                    ∀ τ ∈ L²(Ω)ⁿ
+//                ( ψₖ , ∇v )                = ( -1 , v) + ( ψₖ₋₁ , ∇v )   ∀ v ∈ H¹₀(Ω)
+//
+//              where Zₖ(ψ) = ψ / ( 1/αₖ + |ψ|² )^{1/2} and αₖ > 0.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ZCoefficient : public VectorCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   ZCoefficient(int vdim, GridFunction &psi_, real_t alpha_ = 1.0)
+      : VectorCoefficient(vdim), psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(Vector &V, ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+class DZCoefficient : public MatrixCoefficient
+{
+protected:
+   GridFunction *psi;
+   real_t alpha;
+
+public:
+   DZCoefficient(int height, GridFunction &psi_, real_t alpha_ = 1.0)
+      : MatrixCoefficient(height, true),  psi(&psi_), alpha(alpha_) { }
+
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T, const IntegrationPoint &ip);
+};
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int max_it = 5;
+   int ref_levels = 3;
+   real_t alpha = 1.0;
+   real_t tol = 1e-4;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ref_levels, "-r", "--refs",
+                  "Number of h-refinements.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of iterations");
+   args.AddOption(&tol, "-tol", "--tol",
+                  "Stopping criteria based on the difference between"
+                  "successive solution updates");
+   args.AddOption(&alpha, "-step", "--step",
+                  "Step size alpha.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+
+   MFEM_ASSERT(mesh.bdr_attributes.Size(),
+      "This example does not currently support meshes"
+      " without boundary attributes."
+   )
+
+   // 3. Postprocess the mesh.
+   // 3A. Refine the mesh to increase the resolution.
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 3B. Interpolate the geometry after refinement to control geometry error.
+   // NOTE: Minimum second-order interpolation is used to improve the accuracy.
+   int curvature_order = max(order,2);
+   mesh.SetCurvature(curvature_order);
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   L2_FECollection L2fec(order, dim);
+   FiniteElementSpace L2fes(&mesh, &L2fec, sdim);
+
+   H1_FECollection H1fec(order, dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   cout << "Number of L2 finite element unknowns: "
+        << L2fes.GetTrueVSize() << endl;
+   cout << "Number of H1 finite element unknowns: "
+        << H1fes.GetTrueVSize() << endl;
+
+   // 5. Determine the list of true (i.e., conforming) essential boundary dofs.
+   Array<int> ess_vdof_list;
+   ess_vdof_list.SetSize(H1fes.GetTrueVSize());
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialVDofs(ess_bdr, ess_vdof_list);
+   }
+   
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = L2fes.GetVSize();
+   offsets[2] = H1fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets), rhs(offsets);
+   x = 0.0; rhs = 0.0;
+
+   // 6. Define an initial guess for the solution.
+   ConstantCoefficient one(-1.0);
+   ConstantCoefficient neg_one(-1.0);
+   ConstantCoefficient zero(0.0);
+
+   // 7. Define the solution vectors as a finite element grid functions
+   //    corresponding to the fespaces.
+   GridFunction u_gf, delta_psi_gf;
+
+   delta_psi_gf.MakeRef(&L2fes,x,offsets[0]);
+   u_gf.MakeRef(&H1fes,x,offsets[1]);
+   delta_psi_gf = 0.0;
+
+   GridFunction psi_old_gf(&L2fes);
+   GridFunction psi_gf(&L2fes);
+   GridFunction u_old_gf(&H1fes);
+   u_old_gf = 0.0;
+
+   // 8. Define the function coefficients for the solution and use them to
+   //    initialize the initial guess
+   psi_gf = 0.0;
+   psi_old_gf = psi_gf;
+   u_old_gf = u_gf;
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost,visport);
+      sol_sock.precision(8);
+   }
+
+   // 10. Iterate
+   int k;
+   int total_iterations = 0;
+   real_t increment_u = 0.1;
+   for (k = 0; k < max_it; k++)
+   {
+      GridFunction u_tmp(&H1fes);
+      u_tmp = u_old_gf;
+
+      mfem::out << "\nOUTER ITERATION " << k+1 << endl;
+
+      int j;
+      for ( j = 0; j < 5; j++)
+      {
+         total_iterations++;
+
+         ConstantCoefficient alpha_cf(alpha);
+
+         LinearForm b0,b1;
+         b0.Update(&L2fes,rhs.GetBlock(0),0);
+         b1.Update(&H1fes,rhs.GetBlock(1),0);
+
+         ZCoefficient Z(sdim, psi_gf, alpha);
+         DZCoefficient DZ(sdim, psi_gf, alpha);
+
+         ScalarVectorProductCoefficient neg_Z(-1.0, Z);
+         b0.AddDomainIntegrator(new VectorDomainLFIntegrator(neg_Z));
+         b0.Assemble();
+
+         VectorGridFunctionCoefficient psi_cf(&psi_gf);
+         VectorGridFunctionCoefficient psi_old_cf(&psi_old_gf);
+         VectorSumCoefficient psi_old_minus_psi(psi_old_cf, psi_cf, 1.0, -1.0);
+
+         b1.AddDomainIntegrator(new DomainLFIntegrator(neg_one));
+         b1.AddDomainIntegrator(new DomainLFGradIntegrator(psi_old_minus_psi));
+         b1.Assemble();
+
+         BilinearForm a00(&L2fes);
+         a00.AddDomainIntegrator(new VectorMassIntegrator(DZ));
+         // ConstantCoefficient eps(1e-2);
+         // a00.AddDomainIntegrator(new VectorMassIntegrator(eps));
+         a00.Assemble();
+         a00.Finalize();
+         SparseMatrix &A00 = a00.SpMat();
+
+         MixedBilinearForm a01(&H1fes,&L2fes);
+         a01.AddDomainIntegrator(new GradientIntegrator());
+         a01.Assemble();
+         a01.EliminateEssentialBCFromTrialDofs(ess_vdof_list,x.GetBlock(1),rhs.GetBlock(0));
+         a01.Finalize();
+         SparseMatrix &A01 = a01.SpMat();
+
+         SparseMatrix *A10 = Transpose(A01);
+
+         BilinearForm a11(&H1fes);
+         a11.AddDomainIntegrator(new MassIntegrator(zero));
+         a11.Assemble(false);
+         a11.EliminateEssentialBCFromDofs(ess_vdof_list,x.GetBlock(1),rhs.GetBlock(1));
+         a11.Finalize();
+         SparseMatrix &A11 = a11.SpMat();
+
+        //  BlockOperator A(offsets);
+        //  A.SetBlock(0,0,&A00);
+        //  A.SetBlock(1,0,&A10);
+        //  A.SetBlock(0,1,A01);
+
+        //  BlockDiagonalPreconditioner prec(offsets);
+        //  prec.SetDiagonalBlock(0,new GSSmoother(A00));
+        //  prec.SetDiagonalBlock(1,new GSSmoother(A11));
+        //  prec.owns_blocks = 1;
+
+        //  GMRES(A,prec,rhs,x,0,10000,500,1e-12,0.0);
+
+         BlockMatrix A(offsets);
+         A.SetBlock(0,0,&A00);
+         A.SetBlock(1,0,A10);
+         A.SetBlock(0,1,&A01);
+         A.SetBlock(1,1,&A11);
+
+         SparseMatrix * A_mono = A.CreateMonolithic();
+         UMFPackSolver umf(*A_mono);
+         umf.Mult(rhs,x);
+
+         delta_psi_gf.MakeRef(&L2fes, x.GetBlock(0), 0);
+         u_gf.MakeRef(&H1fes, x.GetBlock(1), 0);
+
+         u_tmp -= u_gf;
+         real_t Newton_update_size = u_tmp.ComputeL2Error(zero);
+         u_tmp = u_gf;
+
+         real_t gamma = 0.1;
+         delta_psi_gf *= gamma;
+         psi_gf += delta_psi_gf;
+
+         if (visualization)
+         {
+            // sol_sock << "solution\n" << mesh << psi_gf << "window_title 'Discrete solution'"
+            sol_sock << "solution\n" << mesh << u_gf << "window_title 'Discrete solution'"
+                     << flush;
+            mfem::out << "Newton_update_size = " << Newton_update_size << endl;
+         }
+
+         delete A10;
+
+         if (Newton_update_size < increment_u)
+         {
+            break;
+         }
+      }
+
+      u_tmp = u_gf;
+      u_tmp -= u_old_gf;
+      increment_u = u_tmp.ComputeL2Error(zero);
+
+      mfem::out << "Number of Newton iterations = " << j+1 << endl;
+      mfem::out << "Increment (|| uₕ - uₕ_prvs||) = " << increment_u << endl;
+
+      u_old_gf = u_gf;
+      psi_old_gf = psi_gf;
+
+      if (increment_u < tol || k == max_it-1)
+      {
+         break;
+      }
+
+      // alpha *= 2.0;
+
+   }
+
+   mfem::out << "\n Outer iterations: " << k+1
+             << "\n Total iterations: " << total_iterations
+             << "\n Total dofs:       " << L2fes.GetTrueVSize() + H1fes.GetTrueVSize()
+             << endl;
+
+   return 0;
+}
+
+void ZCoefficient::Eval(Vector &V, ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(vdim);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/alpha + norm*norm);
+
+   V = psi_vals;
+   V *= phi;
+}
+
+void DZCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                                                const IntegrationPoint &ip)
+{
+   MFEM_ASSERT(psi != NULL, "grid function is not set");
+   MFEM_ASSERT(alpha > 0, "alpha is not positive");
+
+   Vector psi_vals(height);
+   psi->GetVectorValue(T, ip, psi_vals);
+   real_t norm = psi_vals.Norml2();
+   real_t phi = 1.0 / sqrt(1.0/alpha + norm*norm);
+
+   K = 0.0;
+   for (int i = 0; i < height; i++)
+   {
+    K(i,i) = phi;
+    for (int j = 0; j < height; j++)
+      {
+        K(i,j) -= psi_vals(i) * psi_vals(j) * pow(phi, 3);
+      }
+   }
+}

examples/makefile

@@ -23,14 +23,14 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
    ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-   ex31 ex33 ex34 ex36 ex37 ex38 ex39 ex40
+   ex31 ex33 ex34 ex36 ex37 ex38 ex39
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
    ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
    ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p ex40p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
-   ex22p ex24p ex25p ex26p ex34p ex35p
+   ex37p ex39p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
+   ex24p ex25p ex26p ex34p ex35p
 
 ifeq ($(MFEM_USE_LAPACK),YES)
    SEQ_EXAMPLES += ex38
@@ -138,14 +138,6 @@ ex10-test-seq: ex10
 	@$(call mfem-test,$<,, Serial example,-tf 5)
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-tf 5)
-ex14-test-seq-cuda: ex14
-	@$(call mfem-test,$<,, Serial CUDA example,-r 2 -pa -d cuda)
-ex14p-test-par-cuda: ex14p
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-rs 2 -rp 0 -pa -d cuda)
-ex14-test-seq-hip: ex14
-	@$(call mfem-test,$<,, Serial HIP example,-r 2 -pa -d hip)
-ex14p-test-par-hip: ex14p
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-rs 2 -rp 0 -pa -d hip)
 ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p

examples/moonolith/example_utils.hpp

@@ -1,14 +1,3 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
 #include <algorithm>
 #include <assert.h>
 #include <cstdlib>

examples/petsc/ex10p.cpp

@@ -709,7 +709,10 @@ real_t HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const
 
 real_t HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   real_t energy = 0.5*M.ParInnerProduct(v, v);
+   real_t loc_energy = 0.5*M.InnerProduct(v, v);
+   real_t energy;
+   MPI_Allreduce(&loc_energy, &energy, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                 fespace.GetComm());
    return energy;
 }
 

examples/sundials/ex10p.cpp

@@ -856,7 +856,10 @@ double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const
 
 double HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const
 {
-   double energy = 0.5*M.ParInnerProduct(v, v);
+   double loc_energy = 0.5*M.InnerProduct(v, v);
+   double energy;
+   MPI_Allreduce(&loc_energy, &energy, 1, MPI_DOUBLE, MPI_SUM,
+                 fespace.GetComm());
    return energy;
 }
 

fem/CMakeLists.txt

@@ -18,7 +18,6 @@ set(SRCS
   integ/bilininteg_convection_pa.cpp
   integ/bilininteg_convection_ea.cpp
   integ/bilininteg_curlcurl_pa.cpp
-  integ/bilininteg_dgdiffusion_pa.cpp
   integ/bilininteg_dgtrace_pa.cpp
   integ/bilininteg_dgtrace_ea.cpp
   integ/bilininteg_diffusion_mf.cpp
@@ -118,7 +117,6 @@ set(SRCS
   quadinterpolator.cpp
   quadinterpolator_face.cpp
   restriction.cpp
-  normal_deriv_restriction.cpp
   staticcond.cpp
   tmop.cpp
   tmop/tmop_pa.cpp
@@ -230,7 +228,6 @@ set(HDRS
   quadinterpolator.hpp
   quadinterpolator_face.hpp
   restriction.hpp
-  normal_deriv_restriction.hpp
   fespacehierarchy.hpp
   staticcond.hpp
   tbilinearform.hpp

fem/bilinearform.hpp

@@ -340,9 +340,9 @@ public:
        $ M^{-1} $ (currently returns NULL) */
    virtual MatrixInverse *Inverse() const;
 
-   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
        AssemblyLevel::LEGACY.
-       The matrix that gets finalized is different if you are using static
+       THe matrix that gets finalized is different if you are using static
        condensation or hybridization.*/
    virtual void Finalize(int skip_zeros = 1);
 
@@ -643,7 +643,7 @@ public:
    void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                        DiagonalPolicy dpolicy = DIAG_ONE);
 
-   /** @brief Eliminate the given @a vdofs, storing the eliminated part
+   /** @brief  Eliminate the given @a vdofs, storing the eliminated part
        internally in $ M_e $.
 
        This method works in conjunction with EliminateVDofsInRHS() and allows
@@ -826,7 +826,7 @@ public:
        $ M^{-1} $ (currently unimplemented and returns NULL)*/
    virtual MatrixInverse *Inverse() const;
 
-   /** @brief Finalizes the matrix initialization if the ::AssemblyLevel is
+   /** @brief  Finalizes the matrix initialization if the ::AssemblyLevel is
        AssemblyLevel::LEGACY.*/
    virtual void Finalize(int skip_zeros = 1);
 

fem/bilinearform_ext.cpp

@@ -282,22 +282,6 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
       int_face_X.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
       int_face_Y.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
       int_face_Y.UseDevice(true); // ensure 'int_face_Y = 0.0' is done on device
-
-      bool needs_normal_derivs = false;
-      auto &integs = *a->GetFBFI();
-      for (int i = 0; i < integs.Size(); ++i)
-      {
-         if (integs[i]->RequiresFaceNormalDerivatives())
-         {
-            needs_normal_derivs = true;
-            break;
-         }
-      }
-      if (needs_normal_derivs)
-      {
-         int_face_dXdn.SetSize(int_face_restrict_lex->Height());
-         int_face_dYdn.SetSize(int_face_restrict_lex->Height());
-      }
    }
 
    const bool has_bdr_integs = (a->GetBFBFI()->Size() > 0 ||
@@ -312,22 +296,6 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
       bdr_face_Y.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
       bdr_face_Y.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device
 
-      bool needs_normal_derivs = false;
-      auto &integs = *a->GetBFBFI();
-      for (int i = 0; i < integs.Size(); ++i)
-      {
-         if (integs[i]->RequiresFaceNormalDerivatives())
-         {
-            needs_normal_derivs = true;
-            break;
-         }
-      }
-      if (needs_normal_derivs)
-      {
-         bdr_face_dXdn.SetSize(bdr_face_restrict_lex->Height());
-         bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height());
-      }
-
       const Mesh &mesh = *trial_fes->GetMesh();
       // See LinearFormExtension::Update for explanation of f_to_be logic.
       std::unordered_map<int,int> f_to_be;
@@ -574,8 +542,8 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          localY = 0.0;
          for (int i = 0; i < iSz; ++i)
          {
-            AddMultWithMarkers(*integrators[i], localX, elem_markers[i],
-                               elem_attributes, false, localY);
+            AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
+                               false, localY);
          }
          elem_restrict->MultTranspose(localY, y);
       }
@@ -589,57 +557,15 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    const int iFISz = intFaceIntegrators.Size();
    if (int_face_restrict_lex && iFISz>0)
    {
-      // When assembling interior face integrators for DG spaces, we need to
-      // exchange the face-neighbor information. This happens inside member
-      // functions of the 'int_face_restrict_lex'. To avoid repeated calls to
-      // ParGridFunction::ExchangeFaceNbrData, if we have a parallel space
-      // with interior face integrators, we create a ParGridFunction that
-      // will be used to cache the face-neighbor data. x_dg should be passed
-      // to any restriction operator that may need to use face-neighbor data.
-      const Vector *x_dg = &x;
-#ifdef MFEM_USE_MPI
-      ParGridFunction x_pgf;
-      if (auto *pfes = dynamic_cast<ParFiniteElementSpace*>(a->FESpace()))
-      {
-         x_pgf.MakeRef(pfes, const_cast<Vector&>(x), 0);
-         x_dg = &x_pgf;
-      }
-#endif
-
-      int_face_restrict_lex->Mult(*x_dg, int_face_X);
-      if (int_face_dXdn.Size() > 0)
-      {
-         int_face_restrict_lex->NormalDerivativeMult(*x_dg, int_face_dXdn);
-      }
-      if (int_face_X.Size() > 0)
+      int_face_restrict_lex->Mult(x, int_face_X);
+      if (int_face_X.Size()>0)
       {
          int_face_Y = 0.0;
-
-         // if normal derivatives are needed by at least one integrator...
-         if (int_face_dYdn.Size() > 0)
-         {
-            int_face_dYdn = 0.0;
-         }
-
          for (int i = 0; i < iFISz; ++i)
          {
-            if (intFaceIntegrators[i]->RequiresFaceNormalDerivatives())
-            {
-               intFaceIntegrators[i]->AddMultPAFaceNormalDerivatives(
-                  int_face_X, int_face_dXdn,
-                  int_face_Y, int_face_dYdn);
-            }
-            else
-            {
-               intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
-            }
+            intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
          }
          int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
-         if (int_face_dYdn.Size() > 0)
-         {
-            int_face_restrict_lex->NormalDerivativeAddMultTranspose(
-               int_face_dYdn, y);
-         }
       }
    }
 
@@ -653,19 +579,9 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
       Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
       bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_dXdn.Size() > 0)
-      {
-         bdr_face_restrict_lex->NormalDerivativeMult(x, bdr_face_dXdn);
-      }
-      if (bdr_face_X.Size() > 0)
+      if (bdr_face_X.Size()>0)
       {
          bdr_face_Y = 0.0;
-
-         // if normal derivatives are needed by at least one integrator...
-         if (bdr_face_dYdn.Size() > 0)
-         {
-            bdr_face_dYdn = 0.0;
-         }
          for (int i = 0; i < n_bdr_integs; ++i)
          {
             AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
@@ -673,23 +589,10 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          }
          for (int i = 0; i < n_bdr_face_integs; ++i)
          {
-            if (bdr_face_integs[i]->RequiresFaceNormalDerivatives())
-            {
-               AddMultNormalDerivativesWithMarkers(
-                  *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn,
-                  bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn);
-            }
-            else
-            {
-               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                                  bdr_attributes, false, bdr_face_Y);
-            }
+            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                               bdr_attributes, false, bdr_face_Y);
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
-         if (bdr_face_dYdn.Size() > 0)
-         {
-            bdr_face_restrict_lex->NormalDerivativeAddMultTranspose(bdr_face_dYdn, y);
-         }
       }
    }
 }
@@ -790,37 +693,6 @@ static void AddWithMarkers_(
    });
 }
 
-void PABilinearFormExtension::AddMultNormalDerivativesWithMarkers(
-   const BilinearFormIntegrator &integ,
-   const Vector &x,
-   const Vector &dxdn,
-   const Array<int> *markers,
-   const Array<int> &attributes,
-   Vector &y,
-   Vector &dydn) const
-{
-   if (markers)
-   {
-      tmp_evec.SetSize(y.Size() + dydn.Size());
-      tmp_evec = 0.0;
-      Vector tmp_y(tmp_evec, 0, y.Size());
-      Vector tmp_dydn(tmp_evec, y.Size(), dydn.Size());
-
-      integ.AddMultPAFaceNormalDerivatives(x, dxdn, tmp_y, tmp_dydn);
-
-      const int ne = attributes.Size();
-      const int nd_1 = x.Size() / ne;
-      const int nd_2 = dxdn.Size() / ne;
-
-      AddWithMarkers_(ne, nd_1, tmp_y, *markers, attributes, y);
-      AddWithMarkers_(ne, nd_2, tmp_dydn, *markers, attributes, dydn);
-   }
-   else
-   {
-      integ.AddMultPAFaceNormalDerivatives(x, dxdn, y, dydn);
-   }
-}
-
 void PABilinearFormExtension::AddMultWithMarkers(
    const BilinearFormIntegrator &integ,
    const Vector &x,

fem/bilinearform_ext.hpp

@@ -74,8 +74,6 @@ protected:
    mutable Vector localX, localY;
    mutable Vector int_face_X, int_face_Y;
    mutable Vector bdr_face_X, bdr_face_Y;
-   mutable Vector int_face_dXdn, int_face_dYdn;
-   mutable Vector bdr_face_dXdn, bdr_face_dYdn;
    const Operator *elem_restrict; // Not owned
    const FaceRestriction *int_face_restrict_lex; // Not owned
    const FaceRestriction *bdr_face_restrict_lex; // Not owned
@@ -115,23 +113,6 @@ protected:
                            const Array<int> &attributes,
                            const bool transpose,
                            Vector &y) const;
-
-   /// @brief Performs the same function as AddMultWithMarkers, but takes as
-   /// input and output face normal derivatives.
-   ///
-   /// This is required when the integrator requires face normal derivatives,
-   /// for example, DGDiffusionIntegrator.
-   ///
-   /// This is called when the integrator's member function
-   /// BilinearFormIntegrator::RequiresFaceNormalDerivatives() returns true.
-   void AddMultNormalDerivativesWithMarkers(
-      const BilinearFormIntegrator &integ,
-      const Vector &x,
-      const Vector &dxdn,
-      const Array<int> *markers,
-      const Array<int> &attributes,
-      Vector &y,
-      Vector &dydn) const;
 };
 
 /// Data and methods for element-assembled bilinear forms

fem/bilininteg.cpp

@@ -189,12 +189,6 @@ void BilinearFormIntegrator::AssembleTraceFaceMatrix (int elem,
               " Integrator class.");
 }
 
-void BilinearFormIntegrator::AddMultPAFaceNormalDerivatives(
-   const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const
-{
-   MFEM_ABORT("Not implemented.");
-}
-
 void BilinearFormIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
    Vector &elvect)
@@ -774,16 +768,17 @@ void GradientIntegrator::AssembleElementMatrix2(
    ElementTransformation &Trans,  DenseMatrix &elmat)
 {
    dim = test_fe.GetDim();
+   int spaceDim = Trans.GetSpaceDim();
    int trial_dof = trial_fe.GetDof();
    int test_dof = test_fe.GetDof();
    real_t c;
    Vector d_col;
 
    dshape.SetSize(trial_dof, dim);
-   gshape.SetSize(trial_dof, dim);
-   Jadj.SetSize(dim);
+   gshape.SetSize(trial_dof, spaceDim);
+   Jadj.SetSize(dim, spaceDim);
    shape.SetSize(test_dof);
-   elmat.SetSize(dim * test_dof, trial_dof);
+   elmat.SetSize(spaceDim * test_dof, trial_dof);
 
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
                                                             Trans);
@@ -810,7 +805,7 @@ void GradientIntegrator::AssembleElementMatrix2(
       }
       shape *= c;
 
-      for (int d = 0; d < dim; ++d)
+      for (int d = 0; d < spaceDim; ++d)
       {
          gshape.GetColumnReference(d, d_col);
          MultVWt(shape, d_col, elmat_comp);
@@ -3429,7 +3424,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
    const FiniteElement &el1, const FiniteElement &el2,
    FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   int ndof1, ndof2, ndofs;
+   int dim, ndof1, ndof2, ndofs;
    bool kappa_is_nonzero = (kappa != 0.);
    real_t w, wq = 0.0;
 
@@ -3472,9 +3467,17 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      const int order = (ndof2) ? max(el1.GetOrder(),
-                                      el2.GetOrder()) : el1.GetOrder();
-      ir = &GetRule(order, Trans);
+      // a simple choice for the integration order; is this OK?
+      int order;
+      if (ndof2)
+      {
+         order = 2*max(el1.GetOrder(), el2.GetOrder());
+      }
+      else
+      {
+         order = 2*el1.GetOrder();
+      }
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
    }
 
    // assemble: < {(Q \nabla u).n},[v] >      --> elmat
@@ -3652,13 +3655,6 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
    }
 }
 
-const IntegrationRule &DGDiffusionIntegrator::GetRule(
-   int order, FaceElementTransformations &T)
-{
-   // order is typically the maximum of the order of the left and right elements
-   // neighboring the given face.
-   return IntRules.Get(T.GetGeometryType(), 2*order);
-}
 
 // static method
 void DGElasticityIntegrator::AssembleBlock(

fem/bilininteg.hpp

@@ -266,39 +266,6 @@ public:
                                     Vector &flux, Vector *d_energy = NULL)
    { return 0.0; }
 
-   /** @brief For bilinear forms on element faces, specifies if the normal
-              derivatives are needed on the faces or just the face restriction.
-
-       @details if RequiresFaceNormalDerivatives() == true, then
-                AddMultPAFaceNormalDerivatives(...) should be invoked in place
-                of AddMultPA(...) and L2NormalDerivativeFaceRestriction should
-                be used to compute the normal derivatives. This is used for some
-                DG integrators, for example DGDiffusionIntegrator.
-
-       @returns whether normal derivatives appear in the bilinear form.
-   */
-   virtual bool RequiresFaceNormalDerivatives() const { return false; }
-
-   /// Method for partially assembled action.
-   /** @brief For bilinear forms on element faces that depend on the normal
-              derivative on the faces, computes the action of integrator to the
-              face values @a x and reference-normal derivatives @a dxdn and adds
-              the result to @a y and @a dydn.
-
-      @details This method can be called only after the method AssemblePA() has
-               been called.
-
-      @param[in]     x E-vector of face values (provided by
-                       FaceRestriction::Mult)
-      @param[in]     dxdn E-vector of face reference-normal derivatives
-                          (provided by FaceRestriction::NormalDerivativeMult)
-      @param[in,out] y E-vector of face values to add action to.
-      @param[in,out] dydn E-vector of face reference-normal derivative values to
-                          add action to.
-   */
-   virtual void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
-                                               Vector &y, Vector &dydn) const;
-
    virtual ~BilinearFormIntegrator() { }
 };
 
@@ -1741,7 +1708,7 @@ public:
    { vector_fe.CalcPhysDShape(Trans, shape); }
 };
 
-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \cdot \nabla u, \nabla \cdot v)$ in 2D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \cdot  \nabla \cdot u, \nabla \cdot v)$ in 2D
     or 3D and where $\hat{V}$ is a vector coefficient, $u$ is in $H^1$ and $v$ is in $H(div)$. */
 class MixedGradDivIntegrator : public MixedScalarVectorIntegrator
 {
@@ -1780,7 +1747,7 @@ public:
    { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };
 
-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \nabla \cdot u, \nabla v)$ in 2D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} \nabla \cdot u, \nabla \cdot v)$ in 2D
     or 3D and where $\hat{V}$ is a vector coefficient, $u$ is in $H(div)$ and $v$ is in $H^1$. */
 class MixedDivGradIntegrator : public MixedScalarVectorIntegrator
 {
@@ -1820,7 +1787,7 @@ public:
    { scalar_fe.CalcPhysDivShape(Trans, shape); }
 };
 
-/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} u, \nabla v)$ in 2D or 3D
+/** Class for integrating the bilinear form $a(u,v) := (-\hat{V} u, \nabla \cdot v)$ in 2D or 3D
     and where $\hat{V}$ is a vector coefficient, $u$ is in $H^1$ or $L_2$ and $v$ is in $H^1$. */
 class MixedScalarWeakDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
@@ -3262,13 +3229,6 @@ protected:
    Vector shape1, shape2, dshape1dn, dshape2dn, nor, nh, ni;
    DenseMatrix jmat, dshape1, dshape2, mq, adjJ;
 
-
-   // PA extension
-   Vector pa_data; // (Q, h, dot(n,J)|el0, dot(n,J)|el1)
-   const DofToQuad *maps; ///< Not owned
-   int dim, nf, nq, dofs1D, quad1D;
-   IntegrationRules irs{0, Quadrature1D::GaussLobatto};
-
 public:
    DGDiffusionIntegrator(const real_t s, const real_t k)
       : Q(NULL), MQ(NULL), sigma(s), kappa(k) { }
@@ -3277,26 +3237,10 @@ public:
    DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k)
       : Q(NULL), MQ(&q), sigma(s), kappa(k) { }
    using BilinearFormIntegrator::AssembleFaceMatrix;
-   void AssembleFaceMatrix(const FiniteElement &el1,
-                           const FiniteElement &el2,
-                           FaceElementTransformations &Trans,
-                           DenseMatrix &elmat) override;
-
-   bool RequiresFaceNormalDerivatives() const override { return true; }
-
-   using BilinearFormIntegrator::AssemblePA;
-
-   void AssemblePAInteriorFaces(const FiniteElementSpace &fes) override;
-
-   void AssemblePABoundaryFaces(const FiniteElementSpace &fes) override;
-
-   void AddMultPAFaceNormalDerivatives(const Vector &x, const Vector &dxdn,
-                                       Vector &y, Vector &dydn) const override;
-
-   const IntegrationRule &GetRule(int order, FaceElementTransformations &T);
-
-private:
-   void SetupPA(const FiniteElementSpace &fes, FaceType type);
+   virtual void AssembleFaceMatrix(const FiniteElement &el1,
+                                   const FiniteElement &el2,
+                                   FaceElementTransformations &Trans,
+                                   DenseMatrix &elmat);
 };
 
 /** Integrator for the "BR2" diffusion stabilization term

fem/coefficient.cpp

@@ -807,7 +807,6 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 
    QuadratureSpaceBase &qspace = *qf.GetSpace();
    const int ne = qspace.GetNE();
-   qf.HostWrite();
    DenseMatrix values;
    DenseSymmetricMatrix matrix;
    for (int iel = 0; iel < ne; ++iel)
@@ -819,7 +818,7 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
       {
          const IntegrationPoint &ip = ir[iq];
          T.SetIntPoint(&ip);
-         matrix.UseExternalData(&values(0, iq), height);
+         matrix.UseExternalData(&values(0, iq), vdim);
          Eval(matrix, T, ip);
       }
    }
@@ -829,12 +828,13 @@ void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
-   Eval(mat_aux, T, ip);
+   mat.SetSize(height);
+   Eval(mat, T, ip);
    for (int j = 0; j < width; ++j)
    {
       for (int i = 0; i < height; ++ i)
       {
-         K(i, j) = mat_aux(i, j);
+         K(i, j) = mat(i, j);
       }
    }
 }
@@ -924,75 +924,6 @@ void MatrixArrayCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
    }
 }
 
-MatrixArrayVectorCoefficient::MatrixArrayVectorCoefficient (int dim)
-   : MatrixCoefficient (dim)
-{
-   Coeff.SetSize(height);
-   ownCoeff.SetSize(height);
-   for (int i = 0; i < height; i++)
-   {
-      Coeff[i] = NULL;
-      ownCoeff[i] = true;
-   }
-}
-
-void MatrixArrayVectorCoefficient::SetTime(real_t t)
-{
-   for (int i=0; i < height; i++)
-   {
-      if (Coeff[i]) { Coeff[i]->SetTime(t); }
-   }
-   this->MatrixCoefficient::SetTime(t);
-}
-
-void MatrixArrayVectorCoefficient::Set(int i, VectorCoefficient * c, bool own)
-{
-   MFEM_ASSERT(i < height && i >= 0, "Row "
-               << i << " does not exist. " <<
-               "Matrix height = " << height << ".");
-   if (ownCoeff[i]) { delete Coeff[i]; }
-   Coeff[i] = c;
-   ownCoeff[i] = own;
-}
-
-MatrixArrayVectorCoefficient::~MatrixArrayVectorCoefficient ()
-{
-   for (int i=0; i < height; i++)
-   {
-      if (ownCoeff[i]) { delete Coeff[i]; }
-   }
-}
-
-void MatrixArrayVectorCoefficient::Eval(int i, Vector &V,
-                                        ElementTransformation &T,
-                                        const IntegrationPoint &ip)
-{
-   MFEM_ASSERT(i < height && i >= 0, "Row "
-               << i << " does not exist. " <<
-               "Matrix height = " << height << ".");
-   if (Coeff[i])
-   {
-      Coeff[i] -> Eval(V, T, ip);
-   }
-   else
-   {
-      V = 0.0;
-   }
-}
-
-void MatrixArrayVectorCoefficient::Eval(DenseMatrix &K,
-                                        ElementTransformation &T,
-                                        const IntegrationPoint &ip)
-{
-   K.SetSize(height, width);
-   Vector V(width);
-   for (int i = 0; i < height; i++)
-   {
-      this->Eval(i, V, T, ip);
-      K.SetRow(i, V);
-   }
-}
-
 void MatrixRestrictedCoefficient::SetTime(real_t t)
 {
    if (c) { c->SetTime(t); }
@@ -1110,27 +1041,6 @@ real_t DeterminantCoefficient::Eval(ElementTransformation &T,
    return ma.Det();
 }
 
-TraceCoefficient::TraceCoefficient(MatrixCoefficient &A)
-   : a(&A), ma(A.GetHeight(), A.GetWidth())
-{
-   MFEM_ASSERT(A.GetHeight() == A.GetWidth(),
-               "TraceCoefficient:  "
-               "Argument must be a square matrix.");
-}
-
-void TraceCoefficient::SetTime(real_t t)
-{
-   if (a) { a->SetTime(t); }
-   this->Coefficient::SetTime(t);
-}
-
-real_t TraceCoefficient::Eval(ElementTransformation &T,
-                              const IntegrationPoint &ip)
-{
-   a->Eval(ma, T, ip);
-   return ma.Trace();
-}
-
 VectorSumCoefficient::VectorSumCoefficient(int dim)
    : VectorCoefficient(dim),
      ACoef(NULL), BCoef(NULL),
@@ -1416,30 +1326,6 @@ void InverseMatrixCoefficient::Eval(DenseMatrix &M,
    M.Invert();
 }
 
-ExponentialMatrixCoefficient::ExponentialMatrixCoefficient(MatrixCoefficient &A)
-   : MatrixCoefficient(A.GetHeight(), A.GetWidth()), a(&A)
-{
-   MFEM_ASSERT(A.GetHeight() == A.GetWidth() && A.GetHeight() == 2,
-               "ExponentialMatrixCoefficient:  "
-               << "Argument must be a square 2x2 matrix."
-               << "  Height = " << A.GetHeight()
-               << ", Width = " << A.GetWidth());
-}
-
-void ExponentialMatrixCoefficient::SetTime(real_t t)
-{
-   if (a) { a->SetTime(t); }
-   this->MatrixCoefficient::SetTime(t);
-}
-
-void ExponentialMatrixCoefficient::Eval(DenseMatrix &M,
-                                        ElementTransformation &T,
-                                        const IntegrationPoint &ip)
-{
-   a->Eval(M, T, ip);
-   M.Exponential();
-}
-
 OuterProductCoefficient::OuterProductCoefficient(VectorCoefficient &A,
                                                  VectorCoefficient &B)
    : MatrixCoefficient(A.GetVDim(), B.GetVDim()), a(&A), b(&B),

fem/coefficient.hpp

@@ -1334,46 +1334,6 @@ public:
    virtual ~MatrixArrayCoefficient();
 };
 
-/** @brief Matrix coefficient defined row-wise by an array of vector
-    coefficients. Rows that are not set will evaluate to zero. The
-    matrix coefficient is stored as an array indexing the rows of
-    the matrix. */
-class MatrixArrayVectorCoefficient : public MatrixCoefficient
-{
-private:
-   Array<VectorCoefficient *> Coeff;
-   Array<bool> ownCoeff;
-
-public:
-   /** @brief Construct a coefficient matrix of dimensions @a dim * @a dim. The
-       actual coefficients still need to be added with Set(). */
-   explicit MatrixArrayVectorCoefficient (int dim);
-
-   /// Set the time for internally stored coefficients
-   void SetTime(real_t t) override;
-
-   /// Get the vector coefficient located at the i-th row of the matrix
-   VectorCoefficient* GetCoeff (int i) { return Coeff[i]; }
-
-   /** @brief Set the coefficient located at the i-th row of the matrix.
-       By this will take ownership of the Coefficient passed in, but this
-       can be overridden with the @a own parameter. */
-   void Set(int i, VectorCoefficient * c, bool own=true);
-
-   using MatrixCoefficient::Eval;
-
-   /// Evaluate coefficient located at the i-th row of the matrix using integration
-   /// point @a ip.
-   void Eval(int i, Vector &V, ElementTransformation &T,
-             const IntegrationPoint &ip);
-
-   /// Evaluate the matrix coefficient @a ip.
-   void Eval(DenseMatrix &K, ElementTransformation &T,
-             const IntegrationPoint &ip) override;
-
-   virtual ~MatrixArrayVectorCoefficient();
-};
-
 
 /** @brief Derived matrix coefficient that has the value of the parent matrix
     coefficient where it is active and is zero otherwise. */
@@ -1466,13 +1426,12 @@ public:
 class SymmetricMatrixCoefficient : public MatrixCoefficient
 {
 protected:
-
    /// Internal matrix used when evaluating this coefficient as a DenseMatrix.
-   mutable DenseSymmetricMatrix mat_aux;
+   DenseSymmetricMatrix mat;
 public:
    /// Construct a dim x dim matrix coefficient.
    explicit SymmetricMatrixCoefficient(int dimension)
-      : MatrixCoefficient(dimension, true), mat_aux(height) { }
+      : MatrixCoefficient(dimension, true) { }
 
    /// Get the size of the matrix.
    int GetSize() const { return height; }
@@ -1505,9 +1464,8 @@ public:
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip);
 
-
-   /// @deprecated Return a reference to the internal matrix used when evaluating this coefficient as a DenseMatrix.
-   MFEM_DEPRECATED const DenseSymmetricMatrix& GetMatrix() { return mat_aux; }
+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
 
    virtual ~SymmetricMatrixCoefficient() { }
 };
@@ -1527,10 +1485,6 @@ public:
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseSymmetricMatrix &M, ElementTransformation &T,
                      const IntegrationPoint &ip) { M = mat; }
-
-   /// Return a reference to the constant matrix.
-   const DenseSymmetricMatrix& GetMatrix() { return mat; }
-
 };
 
 
@@ -1807,31 +1761,6 @@ public:
                        const IntegrationPoint &ip);
 };
 
-/// Scalar coefficient defined as the trace of a matrix coefficient
-class TraceCoefficient : public Coefficient
-{
-private:
-   MatrixCoefficient * a;
-
-   mutable DenseMatrix ma;
-
-public:
-   /// Construct with the matrix.
-   TraceCoefficient(MatrixCoefficient &A);
-
-   /// Set the time for internally stored coefficients
-   void SetTime(real_t t);
-
-   /// Reset the matrix coefficient
-   void SetACoef(MatrixCoefficient &A) { a = &A; }
-   /// Return the matrix coefficient
-   MatrixCoefficient * GetACoef() const { return a; }
-
-   /// Evaluate the trace coefficient at @a ip.
-   virtual real_t Eval(ElementTransformation &T,
-                       const IntegrationPoint &ip);
-};
-
 /// Vector coefficient defined as the linear combination of two vectors
 class VectorSumCoefficient : public VectorCoefficient
 {
@@ -2183,7 +2112,7 @@ public:
                      const IntegrationPoint &ip);
 };
 
-/// Matrix coefficient defined as the transpose of a matrix coefficient
+/// Matrix coefficient defined as the transpose a matrix coefficient
 class TransposeMatrixCoefficient : public MatrixCoefficient
 {
 private:
@@ -2206,7 +2135,7 @@ public:
                      const IntegrationPoint &ip);
 };
 
-/// Matrix coefficient defined as the inverse of a matrix coefficient.
+/// Matrix coefficient defined as the inverse a matrix coefficient.
 class InverseMatrixCoefficient : public MatrixCoefficient
 {
 private:
@@ -2229,29 +2158,6 @@ public:
                      const IntegrationPoint &ip);
 };
 
-/// Matrix coefficient defined as the exponential of a matrix coefficient.
-class ExponentialMatrixCoefficient : public MatrixCoefficient
-{
-private:
-   MatrixCoefficient * a;
-
-public:
-   /// Construct the matrix coefficient.  Result is $ \exp(A) $.
-   ExponentialMatrixCoefficient(MatrixCoefficient &A);
-
-   /// Set the time for internally stored coefficients
-   void SetTime(real_t t);
-
-   /// Reset the matrix coefficient
-   void SetACoef(MatrixCoefficient &A) { a = &A; }
-   /// Return the matrix coefficient
-   MatrixCoefficient * GetACoef() const { return a; }
-
-   /// Evaluate the matrix coefficient at @a ip.
-   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
-                     const IntegrationPoint &ip);
-};
-
 /// Matrix coefficient defined as the outer product of two vector coefficients.
 class OuterProductCoefficient : public MatrixCoefficient
 {

fem/complex_fem.cpp

@@ -1243,16 +1243,25 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
+#if !defined(HYPRE_USING_GPU)
+         ess_tdof_list.HostRead();
+         for (int k = 0; k < n; k++)
+         {
+            const int j = ess_tdof_list[k];
+            Aih->diag->data[Aih->diag->i[j]] = 0.0;
+         }
+#else
          Ah->HypreReadWrite();
          const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(GetHypreMemoryClass(), n);
-         HYPRE_Int *d_diag_i = Aih->diag->i;
+            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+         const int *d_diag_i = Aih->diag->i;
          real_t *d_diag_data = Aih->diag->data;
-         mfem::hypre_forall(n, [=] MFEM_HOST_DEVICE (int k)
+         MFEM_GPU_FORALL(k, n,
          {
             const int j = d_ess_tdof_list[k];
             d_diag_data[d_diag_i[j]] = 0.0;
          });
+#endif
       }
       else
       {

fem/convergence.cpp

@@ -1,18 +1,8 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
 #include "convergence.hpp"
 
 using namespace std;
 
+
 namespace mfem
 {
 

fem/dgmassinv.cpp

@@ -101,7 +101,7 @@ void DGMassInverse::SetRelTol(const real_t rel_tol_) { rel_tol = rel_tol_; }
 
 void DGMassInverse::SetAbsTol(const real_t abs_tol_) { abs_tol = abs_tol_; }
 
-void DGMassInverse::SetMaxIter(const int max_iter_) { max_iter = max_iter_; }
+void DGMassInverse::SetMaxIter(const real_t max_iter_) { max_iter = max_iter_; }
 
 void DGMassInverse::Update()
 {
@@ -137,7 +137,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
 
    const real_t RELTOL = rel_tol;
    const real_t ABSTOL = abs_tol;
-   const int MAXIT = max_iter;
+   const real_t MAXIT = max_iter;
    const bool IT_MODE = iterative_mode;
    const bool CHANGE_BASIS = (d2q != nullptr);
 

fem/dgmassinv.hpp

@@ -96,7 +96,7 @@ public:
    /// Set the absolute tolerance.
    void SetAbsTol(const real_t abs_tol_);
    /// Set the maximum number of iterations.
-   void SetMaxIter(const int max_iter_);
+   void SetMaxIter(const real_t max_iter_);
    /// Recompute operator and preconditioner (when coefficient or mesh changes).
    void Update();
 

fem/fe/face_map_utils.hpp

@@ -13,7 +13,6 @@
 #define MFEM_FACE_MAP_UTILS_HPP
 
 #include "../../general/array.hpp"
-#include "../../general/backends.hpp"
 #include <utility> // std::pair
 #include <vector>
 
@@ -52,189 +51,6 @@ void FillFaceMap(const int n_face_dofs_per_component,
 void GetTensorFaceMap(const int dim, const int order, const int face_id,
                       Array<int> &face_map);
 
-/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
-/// the corresponding DOF index in lexicographic ordering (for a quadrilateral
-/// element).
-MFEM_HOST_DEVICE
-inline int ToLexOrdering2D(const int face_id, const int size1d, const int i)
-{
-   if (face_id==2 || face_id==3)
-   {
-      return size1d-1-i;
-   }
-   else
-   {
-      return i;
-   }
-}
-
-/// @brief Given a face DOF index on a shared face, ordered lexicographically
-/// relative to element 1, return the corresponding face DOF index ordered
-/// lexicographically relative to element 2.
-MFEM_HOST_DEVICE
-inline int PermuteFace2D(const int face_id1, const int face_id2,
-                         const int orientation, const int size1d,
-                         const int index)
-{
-   int new_index;
-   // Convert from element 1 lex ordering to native ordering
-   if (face_id1 == 2 || face_id1 == 3)
-   {
-      new_index = size1d-1-index;
-   }
-   else
-   {
-      new_index = index;
-   }
-   // Permute based on face orientations
-   if (orientation == 1)
-   {
-      new_index = size1d-1-new_index;
-   }
-   // Covert to element 2 lex ordering
-   return ToLexOrdering2D(face_id2, size1d, new_index);
-}
-
-/// @brief Given a face DOF index in native (counter-clockwise) ordering, return
-/// the corresponding DOF index in lexicographic ordering (for a hexahedral
-/// element).
-MFEM_HOST_DEVICE
-inline int ToLexOrdering3D(const int face_id, const int size1d, const int i,
-                           const int j)
-{
-   if (face_id==2 || face_id==1 || face_id==5)
-   {
-      return i + j*size1d;
-   }
-   else if (face_id==3 || face_id==4)
-   {
-      return (size1d-1-i) + j*size1d;
-   }
-   else // face_id==0
-   {
-      return i + (size1d-1-j)*size1d;
-   }
-}
-
-/// @brief Given the index of a face DOF in lexicographic ordering relative
-/// element 1, permute the index so that it is lexicographically ordered
-/// relative to element 2.
-///
-/// The given face corresponds to local face index @a face_id1 relative to
-/// element 1, and @a face_id2 (with @a orientation) relative to element 2.
-MFEM_HOST_DEVICE
-inline int PermuteFace3D(const int face_id1, const int face_id2,
-                         const int orientation,
-                         const int size1d, const int index)
-{
-   int i=0, j=0, new_i=0, new_j=0;
-   i = index%size1d;
-   j = index/size1d;
-   // Convert from lex ordering
-   if (face_id1==3 || face_id1==4)
-   {
-      i = size1d-1-i;
-   }
-   else if (face_id1==0)
-   {
-      j = size1d-1-j;
-   }
-   // Permute based on face orientations
-   switch (orientation)
-   {
-      case 0:
-         new_i = i;
-         new_j = j;
-         break;
-      case 1:
-         new_i = j;
-         new_j = i;
-         break;
-      case 2:
-         new_i = j;
-         new_j = (size1d-1-i);
-         break;
-      case 3:
-         new_i = (size1d-1-i);
-         new_j = j;
-         break;
-      case 4:
-         new_i = (size1d-1-i);
-         new_j = (size1d-1-j);
-         break;
-      case 5:
-         new_i = (size1d-1-j);
-         new_j = (size1d-1-i);
-         break;
-      case 6:
-         new_i = (size1d-1-j);
-         new_j = i;
-         break;
-      case 7:
-         new_i = i;
-         new_j = (size1d-1-j);
-         break;
-   }
-   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
-}
-
-/// @brief Given a face DOF (or quadrature) index ordered lexicographically
-/// relative to element 1, return the associated (i, j) coordinates.
-///
-/// The returned coordinates will be relative to element 1 or element 2
-/// according to the value of side (side == 0 corresponds element 1).
-MFEM_HOST_DEVICE
-inline void FaceIdxToVolIdx2D(const int qi, const int nq, const int face_id0,
-                              const int face_id1, const int side, int &i, int &j)
-{
-   // Note: in 2D, a consistently ordered mesh will always have the element 2
-   // face reversed relative to element 1, so orientation is determined entirely
-   // by side. (In 3D, separate orientation information is needed).
-   const int orientation = side;
-
-   const int face_id = (side == 0) ? face_id0 : face_id1;
-   const int edge_idx = (side == 0) ? qi : PermuteFace2D(face_id0, face_id1,
-                                                         orientation, nq, qi);
-
-   const int level = (face_id == 0 || face_id == 3) ? 0 : (nq-1);
-   const bool x_axis = (face_id == 0 || face_id == 2);
-
-   i = x_axis ? edge_idx : level;
-   j = x_axis ? level : edge_idx;
-}
-
-/// @brief Given a face DOF (or quadrature) index ordered lexicographically
-/// relative to element 1, return the associated (i, j, k) coordinates.
-///
-/// The returned coordinates will be relative to element 1 or element 2
-/// according to the value of side (side == 0 corresponds element 1).
-MFEM_HOST_DEVICE
-inline void FaceIdxToVolIdx3D(const int index, const int size1d,
-                              const int face_id0, const int face_id1,
-                              const int side, const int orientation,
-                              int& i, int& j, int& k)
-{
-   MFEM_VERIFY_KERNEL(face_id1 >= 0 || side == 0,
-                      "Accessing second side but face_id1 is not valid.");
-
-   const int face_id = (side == 0) ? face_id0 : face_id1;
-   const int fidx = (side == 0) ? index
-                    : PermuteFace3D(face_id0, face_id1, orientation, size1d, index);
-
-   const bool xy_plane = (face_id == 0 || face_id == 5);
-   const bool yz_plane = (face_id == 2 || face_id == 4);
-
-   const int level = (face_id == 0 || face_id == 1 || face_id == 4)
-                     ? 0 : (size1d-1);
-
-   const int _i = fidx % size1d;
-   const int _j = fidx / size1d;
-
-   k = xy_plane ? level : _j;
-   j = yz_plane ? _i : xy_plane ? _j : level;
-   i = yz_plane ? level : _i;
-}
-
 } // namespace internal
 
 } // namespace mfem

fem/fe/fe_base.hpp

@@ -316,7 +316,7 @@ public:
    int GetDim() const { return dim; }
 
    /** @brief Returns the vector dimension for vector-valued finite elements,
-       which is also the dimension of the interpolation operation. */
+       which is also the dimension of the interpolation operatrion. */
    int GetRangeDim() const { return vdim; }
 
    /// Returns the dimension of the curl for vector-valued finite elements.

fem/gridfunc.cpp

@@ -1321,9 +1321,9 @@ void GridFunction::ProjectVectorFieldOn(GridFunction &vec_field, int comp)
    }
 }
 
-void GridFunction::AccumulateAndCountDerivativeValues(
-   int comp, int der_comp, GridFunction &der,
-   Array<int> &zones_per_dof) const
+void GridFunction::AccumulateAndCountDerivativeValues(int comp, int der_comp,
+                                                      GridFunction &der,
+                                                      Array<int> &zones_per_dof)
 {
    FiniteElementSpace * der_fes = der.FESpace();
    ElementTransformation * transf;
@@ -1374,8 +1374,7 @@ void GridFunction::AccumulateAndCountDerivativeValues(
    }
 }
 
-void GridFunction::GetDerivative(int comp, int der_comp,
-                                 GridFunction &der) const
+void GridFunction::GetDerivative(int comp, int der_comp, GridFunction &der)
 {
    Array<int> overlap;
    AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -2062,37 +2061,41 @@ void GridFunction::AccumulateAndCountBdrValues(
    Coefficient *coeff[], VectorCoefficient *vcoeff, const Array<int> &attr,
    Array<int> &values_counter)
 {
+   int i, j, fdof, d, ind, vdim;
+   real_t val;
+   const FiniteElement *fe;
+   ElementTransformation *transf;
    Array<int> vdofs;
    Vector vc;
 
    values_counter.SetSize(Size());
    values_counter = 0;
 
-   const int vdim = fes->GetVDim();
+   vdim = fes->GetVDim();
+
    HostReadWrite();
 
-   for (int i = 0; i < fes->GetNBE(); i++)
+   for (i = 0; i < fes->GetNBE(); i++)
    {
       if (attr[fes->GetBdrAttribute(i) - 1] == 0) { continue; }
 
-      const FiniteElement *fe = fes->GetBE(i);
-      const int fdof = fe->GetDof();
-      ElementTransformation *transf = fes->GetBdrElementTransformation(i);
+      fe = fes->GetBE(i);
+      fdof = fe->GetDof();
+      transf = fes->GetBdrElementTransformation(i);
       const IntegrationRule &ir = fe->GetNodes();
       fes->GetBdrElementVDofs(i, vdofs);
 
-      for (int j = 0; j < fdof; j++)
+      for (j = 0; j < fdof; j++)
       {
          const IntegrationPoint &ip = ir.IntPoint(j);
          transf->SetIntPoint(&ip);
          if (vcoeff) { vcoeff->Eval(vc, *transf, ip); }
-         for (int d = 0; d < vdim; d++)
+         for (d = 0; d < vdim; d++)
          {
             if (!vcoeff && !coeff[d]) { continue; }
 
-            real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
-            int ind = vdofs[fdof*d+j];
-            if ( ind < 0 )
+            val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip);
+            if ( (ind = vdofs[fdof*d+j]) < 0 )
             {
                val = -val, ind = -1-ind;
             }
@@ -2114,11 +2117,10 @@ void GridFunction::AccumulateAndCountBdrValues(
    // iff A_ij != 0. It is sufficient to resolve just the first level of
    // dependency, since A is a projection matrix: A^n = A due to cR.cP = I.
    // Cases like these arise in 3D when boundary edges are constrained by
-   // (depend on) internal faces/elements, or for internal boundaries in 2 or
-   // 3D. We use the virtual method GetBoundaryClosure from NCMesh to resolve
-   // the dependencies.
-   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
-                                fes->GetMesh()->Dimension() == 3))
+   // (depend on) internal faces/elements. We use the virtual method
+   // GetBoundaryClosure from NCMesh to resolve the dependencies.
+
+   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
    {
       Vector vals;
       Mesh *mesh = fes->GetMesh();
@@ -2126,19 +2128,26 @@ void GridFunction::AccumulateAndCountBdrValues(
       Array<int> bdr_edges, bdr_vertices, bdr_faces;
       ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges, bdr_faces);
 
-      auto mark_dofs = [&](ElementTransformation &transf, const FiniteElement &fe)
+      for (i = 0; i < bdr_edges.Size(); i++)
       {
+         int edge = bdr_edges[i];
+         fes->GetEdgeVDofs(edge, vdofs);
+         if (vdofs.Size() == 0) { continue; }
+
+         transf = mesh->GetEdgeTransformation(edge);
+         transf->Attribute = -1; // TODO: set the boundary attribute
+         fe = fes->GetEdgeElement(edge);
          if (!vcoeff)
          {
-            vals.SetSize(fe.GetDof());
-            for (int d = 0; d < vdim; d++)
+            vals.SetSize(fe->GetDof());
+            for (d = 0; d < vdim; d++)
             {
                if (!coeff[d]) { continue; }
 
-               fe.Project(*coeff[d], transf, vals);
+               fe->Project(*coeff[d], *transf, vals);
                for (int k = 0; k < vals.Size(); k++)
                {
-                  const int ind = vdofs[d*vals.Size()+k];
+                  ind = vdofs[d*vals.Size()+k];
                   if (++values_counter[ind] == 1)
                   {
                      (*this)(ind) = vals(k);
@@ -2152,11 +2161,11 @@ void GridFunction::AccumulateAndCountBdrValues(
          }
          else // vcoeff != NULL
          {
-            vals.SetSize(vdim*fe.GetDof());
-            fe.Project(*vcoeff, transf, vals);
+            vals.SetSize(vdim*fe->GetDof());
+            fe->Project(*vcoeff, *transf, vals);
             for (int k = 0; k < vals.Size(); k++)
             {
-               const int ind = vdofs[k];
+               ind = vdofs[k];
                if (++values_counter[ind] == 1)
                {
                   (*this)(ind) = vals(k);
@@ -2167,26 +2176,6 @@ void GridFunction::AccumulateAndCountBdrValues(
                }
             }
          }
-      };
-
-      for (auto edge : bdr_edges)
-      {
-         fes->GetEdgeVDofs(edge, vdofs);
-         if (vdofs.Size() == 0) { continue; }
-
-         ElementTransformation *transf = mesh->GetEdgeTransformation(edge);
-         const FiniteElement *fe = fes->GetEdgeElement(edge);
-         mark_dofs(*transf, *fe);
-      }
-
-      for (auto face : bdr_faces)
-      {
-         fes->GetFaceVDofs(face, vdofs);
-         if (vdofs.Size() == 0) { continue; }
-
-         ElementTransformation *transf = mesh->GetFaceTransformation(face);
-         const FiniteElement *fe = fes->GetFaceElement(face);
-         mark_dofs(*transf, *fe);
       }
    }
 }
@@ -2239,37 +2228,26 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
       accumulate_dofs(dofs, lvec, *this, values_counter);
    }
 
-   if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 ||
-                                fes->GetMesh()->Dimension() == 3))
+   if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3)
    {
       Mesh *mesh = fes->GetMesh();
       NCMesh *ncmesh = mesh->ncmesh;
       Array<int> bdr_edges, bdr_vertices, bdr_faces;
       ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges, bdr_faces);
 
-      for (auto edge : bdr_edges)
+      for (int i = 0; i < bdr_edges.Size(); i++)
       {
+         int edge = bdr_edges[i];
          fes->GetEdgeDofs(edge, dofs);
          if (dofs.Size() == 0) { continue; }
 
          T = mesh->GetEdgeTransformation(edge);
+         T->Attribute = -1; // TODO: set the boundary attribute
          fe = fes->GetEdgeElement(edge);
          lvec.SetSize(fe->GetDof());
          fe->Project(vcoeff, *T, lvec);
          accumulate_dofs(dofs, lvec, *this, values_counter);
       }
-
-      for (auto face : bdr_faces)
-      {
-         fes->GetFaceDofs(face, dofs);
-         if (dofs.Size() == 0) { continue; }
-
-         T = mesh->GetFaceTransformation(face);
-         fe = fes->GetFaceElement(face);
-         lvec.SetSize(fe->GetDof());
-         fe->Project(vcoeff, *T, lvec);
-         accumulate_dofs(dofs, lvec, *this, values_counter);
-      }
    }
 }
 

fem/gridfunc.hpp

@@ -321,7 +321,7 @@ public:
        @param[out] der       The resulting derivative (scalar function). The
                              FiniteElementSpace of this function must be set
                              before the call. */
-   void GetDerivative(int comp, int der_comp, GridFunction &der) const;
+   void GetDerivative(int comp, int der_comp, GridFunction &der);
 
    real_t GetDivergence(ElementTransformation &tr) const;
 
@@ -443,7 +443,7 @@ protected:
        GetDerivative() method; see its documentation. */
    void AccumulateAndCountDerivativeValues(int comp, int der_comp,
                                            GridFunction &der,
-                                           Array<int> &zones_per_dof) const;
+                                           Array<int> &zones_per_dof);
 
    void AccumulateAndCountBdrValues(Coefficient *coeff[],
                                     VectorCoefficient *vcoeff,

fem/gslib.cpp

@@ -1352,85 +1352,6 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
    Interpolate(field_in, field_out);
 }
 
-GSOPGSLIB::GSOPGSLIB(Array<long long> &ids)
-{
-   gsl_comm = new gslib::comm;
-   cr       = new gslib::crystal;
-#ifdef MFEM_USE_MPI
-   int initialized;
-   MPI_Initialized(&initialized);
-   if (!initialized) { MPI_Init(NULL, NULL); }
-   MPI_Comm comm = MPI_COMM_WORLD;
-   comm_init(gsl_comm, comm);
-#else
-   comm_init(gsl_comm, 0);
-#endif
-   crystal_init(cr, gsl_comm);
-   UpdateIdentifiers(ids);
-}
-
-#ifdef MFEM_USE_MPI
-GSOPGSLIB::GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids)
-   : cr(NULL), gsl_comm(NULL)
-{
-   gsl_comm = new gslib::comm;
-   cr      = new gslib::crystal;
-   comm_init(gsl_comm, comm_);
-   crystal_init(cr, gsl_comm);
-   UpdateIdentifiers(ids);
-}
-#endif
-
-GSOPGSLIB::~GSOPGSLIB()
-{
-   crystal_free(cr);
-   gslib_gs_free(gsl_data);
-   comm_free(gsl_comm);
-   delete gsl_comm;
-   delete cr;
-}
-
-void GSOPGSLIB::UpdateIdentifiers(const Array<long long> &ids)
-{
-   long long minval = ids.Min();
-#ifdef MFEM_USE_MPI
-   MPI_Allreduce(MPI_IN_PLACE, &minval, 1, MPI_LONG_LONG_INT,
-                 MPI_MIN, gsl_comm->c);
-#endif
-   MFEM_VERIFY(minval >= 0, "Unique identifier cannot be negative.");
-   if (gsl_data != NULL) { gslib_gs_free(gsl_data); }
-   num_ids = ids.Size();
-   gsl_data = gslib_gs_setup(ids.GetData(),
-                             ids.Size(),
-                             gsl_comm, 0,
-                             gslib::gs_crystal_router, 0);
-}
-
-void GSOPGSLIB::GS(Vector &senddata, GSOp op)
-{
-   MFEM_VERIFY(senddata.Size() == num_ids,
-               "Incompatible setup and GOP operation.");
-   if (op == GSOp::ADD)
-   {
-      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_add,0,gsl_data,0);
-   }
-   else if (op == GSOp::MUL)
-   {
-      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_mul,0,gsl_data,0);
-   }
-   else if (op == GSOp::MAX)
-   {
-      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_max,0,gsl_data,0);
-   }
-   else if (op == GSOp::MIN)
-   {
-      gslib_gs(senddata.GetData(),gslib::gs_double,gslib::gs_min,0,gsl_data,0);
-   }
-   else
-   {
-      MFEM_ABORT("Invalid GSOp operation.");
-   }
-}
 
 } // namespace mfem
 

fem/gslib.hpp

@@ -23,16 +23,13 @@ struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
 struct crystal;
-struct gs_data;
 }
 
 namespace mfem
 {
 
 /** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
- *  collection of points.
- *
- *  There are three key functions in FindPointsGSLIB:
+ *  collection of points. There are three key functions in FindPointsGSLIB:
  *
  *  1. Setup - constructs the internal data structures of gslib.
  *
@@ -229,7 +226,6 @@ public:
 
 /** \brief OversetFindPointsGSLIB enables use of findpts for arbitrary number of
     overlapping grids.
-
     The parameters in this class are the same as FindPointsGSLIB with the
     difference of additional inputs required to account for more than 1 mesh. */
 class OversetFindPointsGSLIB : public FindPointsGSLIB
@@ -294,63 +290,6 @@ public:
    using FindPointsGSLIB::Interpolate;
 };
 
-/** \brief  Class for gather-scatter (gs) operations on Vectors based on
-    corresponding global identifiers.
-
-    This functionality is useful for gs-ops on DOF values across processor
-    boundary, where the global identifier would be the corresponding true DOF
-    index. Operations currently supported are min, max, sum, and multiplication.
-    Note: identifier 0 does not participate in the gather-scatter operation and
-    a given identifier can be included multiple times on a given rank.
-    For example, consider a vector, v:
-    - v = [0.3, 0.4, 0.25, 0.7] on rank1,
-    - v = [0.6, 0.1] on rank 2,
-    - v = [-0.2, 0.3, 0.7, 0.] on rank 3.
-
-    Consider a corresponding Array<int>, a:
-    - a = [1, 2, 3, 1] on rank 1,
-    - a = [3, 2] on rank 2,
-    - a = [1, 2, 0, 3] on rank 3.
-
-    A gather-scatter "minimum" operation, done as follows:
-    GSOPGSLIB gs = GSOPGSLIB(MPI_COMM_WORLD, a);
-    gs.GS(v, GSOp::MIN);
-    would return into v:
-    - v = [-0.2, 0.1, 0., -0.2] on rank 1,
-    - v = [0., 0.1] on rank 2,
-    - v = [-0.2, 0.1, 0.7, 0.] on rank 3,
-    where the values have been compared across all processors based on the
-    integer identifier. */
-class GSOPGSLIB
-{
-protected:
-   struct gslib::crystal *cr;               // gslib's internal data
-   struct gslib::comm *gsl_comm;            // gslib's internal data
-   struct gslib::gs_data *gsl_data = NULL;
-   int num_ids;
-
-public:
-   GSOPGSLIB(Array<long long> &ids);
-
-#ifdef MFEM_USE_MPI
-   GSOPGSLIB(MPI_Comm comm_, Array<long long> &ids);
-#endif
-
-   virtual ~GSOPGSLIB();
-
-   /// Supported operation types. See class description.
-   enum GSOp {ADD, MUL, MIN, MAX};
-
-   /// Update the identifiers used for the gather-scatter operator.
-   /// Same @a ids get grouped together and id == 0 does not participate.
-   /// See class description.
-   void UpdateIdentifiers(const Array<long long> &ids);
-
-   /// Gather-Scatter operation on senddata. Must match length of unique
-   /// identifiers used in the constructor. See class description.
-   void GS(Vector &senddata, GSOp op);
-};
-
 } // namespace mfem
 
 #endif // MFEM_USE_GSLIB

fem/hyperbolic.cpp

@@ -18,6 +18,7 @@
 namespace mfem
 {
 
+
 void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
                                                      ElementTransformation &Tr,
                                                      const Vector &elfun,
@@ -28,7 +29,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
    const int dof = el.GetDof();
 
 #ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
+   // Local storages for element integration
 
    // shape function value at an integration point
    Vector shape(dof);
@@ -61,7 +62,7 @@ void HyperbolicFormIntegrator::AssembleElementVector(const FiniteElement &el,
       ir = &IntRules.Get(Tr.GetGeometryType(), order);
    }
 
-   // loop over integration points
+   // loop over interation points
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
@@ -91,7 +92,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    const int dof2 = el2.GetDof();
 
 #ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
+   // Local storages for element integration
 
    // shape function value at an integration point - first elem
    Vector shape1(dof1);
@@ -121,7 +122,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    DenseMatrix elvect2_mat(elvect.GetData() + dof1 * num_equations, dof2,
                            num_equations);
 
-   // Obtain integration rule. If integration is rule is given, then use it.
+   // obtain integration rule. If integration is rule is given, then use it.
    // Otherwise, get (2*p + IntOrderOffset) order integration rule
    const IntegrationRule *ir = IntRule;
    if (!ir)
@@ -148,7 +149,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
       if (nor.Size() == 1)  // if 1D, use 1 or -1.
       {
          // This assume the 1D integration point is in (0,1). This may not work
-         // if this changes.
+         // if this chages.
          nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
       }
       else

fem/hyperbolic.hpp

@@ -18,36 +18,43 @@
 namespace mfem
 {
 
-// This file contains general hyperbolic conservation element/face form
-// integrators. HyperbolicFormIntegrator and RiemannSolver are defined.
+//                     MFEM Hyperbolic Conservation Laws
 //
-// HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
-// element weak divergence and interface flux
+// Description:
 //
-//     ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
+//    This file contains general hyperbolic conservation element/face form
+//    integrators.
 //
-// Here, T is an element, e is an edge, and [[⋅]] is jump. This form integrator
-// is coupled with RiemannSolver that implements the numerical flux F̂. For
-// RiemannSolver, the Rusanov flux, also known as local Lax-Friedrichs flux, is
-// provided.
+//    HyperbolicFormIntegrator and RiemannSolver are defined.
+//    HyperbolicFormIntegrator is a NonlinearFormIntegrator that implements
+//    element weak divergence and interface flux
 //
-// To implement a specific hyperbolic conservation laws, users can create
-// derived classes from FluxFunction with overloaded ComputeFlux. One can
-// optionally overload ComputeFluxDotN to avoid creating dense matrix when
-// computing normal flux. Several example equations are also defined including:
-// advection, Burgers', shallow water, and Euler equations. Users can control
-// the quadrature rule by either providing the integration rule, or integration
-// order offset. Integration will use 2*p + IntOrderOffset order quadrature
-// rule.
+//       ∫_T F(u):∇v,   -∫_e F̂(u)⋅[[v]]
 //
-// At each call of HyperbolicFormIntegrator::AssembleElementVector
-// HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
-// speed will be updated. This will not be reinitialized automatically. To
-// reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See, ex18.hpp.
+//    Here, T is an element, e is an edge, and [[⋅]] is jump. This form
+//    integrator is coupled with RiemannSolver that implements the numerical
+//    flux F̂. For RiemannSolver, the Rusanov flux, also known as local
+//    Lax-Friedrichs flux, is provided.
+//
+//    To implement a specific hyperbolic conservation laws, users can create
+//    derived classes from FluxFunction with overloaded ComputeFlux. One can
+//    optionally overload ComputeFluxDotN to avoid creating dense matrix when
+//    computing normal flux. Several example equations are also defined
+//    including: advection, Burgers', shallow water, and Euler equations. Users
+//    can control the quadrature rule by either providing the integration rule,
+//    or integration order offset. Integration will use 2*p + IntOrderOffset
+//    order quadrature rule.
+//
+//    At each call of HyperbolicFormIntegrator::AssembleElementVector
+//    HyperbolicFormIntegrator::AssembleFaceVector, the maximum characteristic
+//    speed will be updated. This will not be reinitialized automatically.
+//    To reinitialize, use HyperbolicFormIntegrator::ResetMaxCharSpeed. See,
+//    ex18.hpp.
+//
+//    Note: To avoid communication overhead, we update the maximum
+//    characteristic speed within each process. Use a proper MPI routine to
+//    gather the information.
 //
-// Note: To avoid communication overhead, we update the maximum characteristic
-// speed within each MPI process only. Use the appropriate MPI routine to gather
-// the information.
 
 /**
  * @brief Abstract class for hyperbolic flux for a system of hyperbolic
@@ -81,7 +88,7 @@ public:
    virtual real_t ComputeFlux(const Vector &state, ElementTransformation &Tr,
                               DenseMatrix &flux) const = 0;
    /**
-    * @brief Compute normal flux. Optionally overloaded in the
+    * @brief Compute normal flux. Optionally overloadded in the
     * derived class to avoid creating full dense matrix for flux.
     *
     * @param[in] state state at the current integration point
@@ -161,13 +168,13 @@ protected:
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
-   // The maximum characteristic speed, updated during element/face vector assembly
+   // The maximum characterstic speed, updated during element/face vector assembly
    real_t max_char_speed;
    const RiemannSolver &rsolver;   // Numerical flux that maps F(u±,x) to hat(F)
    const FluxFunction &fluxFunction;
    const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
 #ifndef MFEM_THREAD_SAFE
-   // Local storage for element integration
+   // Local storages for element integration
    Vector shape;              // shape function value at an integration point
    Vector state;              // state value at an integration point
    DenseMatrix flux;          // flux value at an integration point

fem/integ/bilininteg_diffusion_patch.cpp

@@ -563,7 +563,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
    cdofs.SetSize(maxw[0], maxw[1], maxw[2]);
 
    // Compute sparsity of the sparse matrix
-   smati = Memory<int>(ndof+1);
+   smati = new int[ndof+1];
    smati[0] = 0;
 
    for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -586,8 +586,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_fullQuadrature(
       nnz += ndd;
    }
 
-   smatj = Memory<int>(nnz);
-   smata = Memory<real_t>(nnz);
+   smatj = new int[nnz];
+   smata = new real_t[nnz];
 
    for (int i=0; i<nnz; ++i)
    {
@@ -973,7 +973,7 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
    cdofs.SetSize(maxw[0], maxw[1], maxw[2]);
 
    // Compute sparsity of the sparse matrix
-   smati = Memory<int>(ndof+1);
+   smati = new int[ndof+1];
    smati[0] = 0;
 
    for (int dof_j=0; dof_j<ndof; ++dof_j)
@@ -996,8 +996,8 @@ void DiffusionIntegrator::AssemblePatchMatrix_reducedQuadrature(
       nnz += ndd;
    }
 
-   smatj = Memory<int>(nnz);
-   smata = Memory<real_t>(nnz);
+   smatj = new int[nnz];
+   smata = new real_t[nnz];
 
    for (int i=0; i<nnz; ++i)
    {

fem/integ/bilininteg_elasticity_kernels.hpp

@@ -157,7 +157,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
    static constexpr int aSize = aUpper-aLower;
    static constexpr bool isComponent = (i_block >= 0);
 
-   // Assuming all elements are the same
+   //Assuming all elements are the same
    const auto &ir = QVec.GetIntRule(0);
    const QuadratureInterpolator *E_To_Q_Map = fespace.GetQuadratureInterpolator(
                                                  ir);
@@ -180,7 +180,7 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
          auto invJ = inv(make_tensor<d, d>(
          [&](int i, int j) { return J(p, i, j, e); }));
          tensor<real_t, aSize, d> gradx;
-         // load grad(x) into gradx
+         //load grad(x) into gradx
          if (isComponent)
          {
             for (int i = 0; i < d; i++)
@@ -198,11 +198,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
                }
             }
          }
-         // compute divergence
+         //compute divergence
          real_t div = 0.;
          for (int i = aLower; i < aUpper; i++)
          {
-            // take size of gradx into account
+            //take size of gradx into account
             const int iIndex = isComponent ? 0 : i;
             div += gradx(iIndex,i);
          }
@@ -211,11 +211,11 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
          {
             for (int q = qLower; q < qUpper; q++)
             {
-               // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-               // this contraction could be made slightly cheaper using Voigt
-               // notation, but repeated entries are summed for simplicity.
+               //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+               //this contraction could be made slightly cheaper using Voigt
+               //notation, but repeated entries are summed for simplicity.
                real_t contraction = 0.;
-               // not sure how to combine cases
+               //not sure how to combine cases
                if (isComponent)
                {
                   for (int a = 0; a < d; a++)
@@ -276,7 +276,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
                                    const CoefficientVector &mu, const GeometricFactors &geom,
                                    const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
-   // Assuming all elements are the same
+   //Assuming all elements are the same
    const auto &ir = QVec.GetIntRule(0);
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
@@ -299,9 +299,9 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
             {
                for (int q = 0; q < d; q++)
                {
-                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                  // this contraction could be made slightly cheaper using Voigt
-                  // notation, but repeated entries are summed for simplicity.
+                  //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  //this contraction could be made slightly cheaper using Voigt
+                  //notation, but repeated entries are summed for simplicity.
                   real_t contraction = 0.;
                   for (int a = 0; a < d; a++)
                   {
@@ -321,7 +321,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
       }
    });
 
-   // Reduce quadrature function to an E-Vector
+   //Reduce quadrature function to an E-Vector
    const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
    auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
    const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
@@ -348,7 +348,7 @@ void ElasticityAssembleDiagonalPA_(const int nDofs,
    });
 }
 
-// Templated implementation of ElasticityAssembleEA.
+//Templated implementation of ElasticityAssembleEA.
 template<int dim>
 void ElasticityAssembleEA_(const int i_block,
                            const int j_block,
@@ -360,7 +360,7 @@ void ElasticityAssembleEA_(const int i_block,
                            const DofToQuad &maps,
                            Vector &emat)
 {
-   // Assuming all elements are the same
+   //Assuming all elements are the same
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
    const int numEls = lambda.Size()/numPoints;
@@ -386,7 +386,7 @@ void ElasticityAssembleEA_(const int i_block,
                {
                   for (int m = 0; m < d; m++)
                   {
-                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                     //compute contraction of 4*sym(grad(u))sym(grad(v)) term.
                      real_t contraction = 0.;
                      for (int a = 0; a < d; a++)
                      {

fem/intrules_cut.cpp

@@ -101,7 +101,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
          }
       }
 
-      // assemble the matrix
+      // assamble the matrix
       DenseMatrix Mat(nBasisVolume, ir.GetNPoints());
       for (int ip = 0; ip < ir.GetNPoints(); ip++)
       {
@@ -118,7 +118,7 @@ void MomentFittingIntRules::InitVolume(int order, Coefficient& levelset,
          Mat.SetCol(ip, shape);
       }
 
-      // compute the SVD for the matrix
+      // compute the svd for the matrix
       VolumeSVD = new DenseMatrixSVD(Mat, 'A', 'A');
       VolumeSVD->Eval(Mat);
    }
@@ -1239,7 +1239,7 @@ void MomentFittingIntRules::OrthoBasis2D(const IntegrationPoint& ip,
 
    shape.SetSize(nBasis, 2);
 
-   // evaluate basis in the point
+   // evaluate basis inthe point
    DenseMatrix preshape(nBasis, 2);
    DivFreeBasis2D(ip, shape);
 
@@ -1597,6 +1597,6 @@ void MomentFittingIntRules::GetSurfaceWeights(ElementTransformation& Tr,
    }
 }
 
-#endif // MFEM_USE_LAPACK
+#endif //MFEM_USE_LAPACK
 
 }

fem/lor/lor_ams.cpp

@@ -291,7 +291,7 @@ void BatchedLOR_AMS::FormCoordinateVectors(const Vector &X_vert)
    const auto ltdof_ldof = HypreRead(R->GetMemoryJ());
 
    // Go from E-vector format directly to T-vector format
-   mfem::hypre_forall(ntdofs, [=] MFEM_HOST_DEVICE (int i)
+   MFEM_HYPRE_FORALL(i, ntdofs,
    {
       const int j = d_offsets[ltdof_ldof[i]];
       for (int c = 0; c < sdim; ++c)

fem/lor/lor_h1_impl.hpp

@@ -269,13 +269,13 @@ void BatchedLOR_H1::Assemble3D()
                real_t vx[8], vy[8], vz[8];
                LORVertexCoordinates3D<ORDER>(X, iel_ho, kx, ky, kz, vx, vy, vz);
 
-               // MFEM_UNROLL(2)
+               //MFEM_UNROLL(2)
                for (int iqz=0; iqz<2; ++iqz)
                {
-                  // MFEM_UNROLL(2)
+                  //MFEM_UNROLL(2)
                   for (int iqy=0; iqy<2; ++iqy)
                   {
-                     // MFEM_UNROLL(2)
+                     //MFEM_UNROLL(2)
                      for (int iqx=0; iqx<2; ++iqx)
                      {
                         const real_t x = iqx;
@@ -307,21 +307,21 @@ void BatchedLOR_H1::Assemble3D()
                   }
                }
 
-               // MFEM_UNROLL(2)
+               //MFEM_UNROLL(2)
                for (int iqx=0; iqx<2; ++iqx)
                {
-                  // MFEM_UNROLL(2)
+                  //MFEM_UNROLL(2)
                   for (int jz=0; jz<2; ++jz)
                   {
                      // Note loop starts at iz=jz here, taking advantage of
                      // symmetries.
-                     // MFEM_UNROLL(2)
+                     //MFEM_UNROLL(2)
                      for (int iz=jz; iz<2; ++iz)
                      {
-                        // MFEM_UNROLL(2)
+                        //MFEM_UNROLL(2)
                         for (int iqy=0; iqy<2; ++iqy)
                         {
-                           // MFEM_UNROLL(2)
+                           //MFEM_UNROLL(2)
                            for (int iqz=0; iqz<2; ++iqz)
                            {
                               const real_t mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
@@ -356,10 +356,10 @@ void BatchedLOR_H1::Assemble3D()
                               real_t wdetJ = Q(6,iqz,iqy,iqx);
                               mass_A(iqy,iz,jz,iqx) += mq*wdetJ*biz*bjz;
                            }
-                           // MFEM_UNROLL(2)
+                           //MFEM_UNROLL(2)
                            for (int jy=0; jy<2; ++jy)
                            {
-                              // MFEM_UNROLL(2)
+                              //MFEM_UNROLL(2)
                               for (int iy=0; iy<2; ++iy)
                               {
                                  const real_t biy = (iy == iqy) ? 1.0 : 0.0;
@@ -382,16 +382,16 @@ void BatchedLOR_H1::Assemble3D()
                               }
                            }
                         }
-                        // MFEM_UNROLL(2)
+                        //MFEM_UNROLL(2)
                         for (int jy=0; jy<2; ++jy)
                         {
-                           // MFEM_UNROLL(2)
+                           //MFEM_UNROLL(2)
                            for (int jx=0; jx<2; ++jx)
                            {
-                              // MFEM_UNROLL(2)
+                              //MFEM_UNROLL(2)
                               for (int iy=0; iy<2; ++iy)
                               {
-                                 // MFEM_UNROLL(2)
+                                 //MFEM_UNROLL(2)
                                  for (int ix=0; ix<2; ++ix)
                                  {
                                     const real_t bix = (ix == iqx) ? 1.0 : 0.0;
@@ -431,7 +431,7 @@ void BatchedLOR_H1::Assemble3D()
                // Assemble the local matrix into the macro-element sparse matrix
                // in a format similar to coordinate format. The (I,J) arrays
                // are implicit (not stored explicitly).
-               // MFEM_UNROLL(8)
+               //MFEM_UNROLL(8)
                for (int ii_loc=0; ii_loc<nv; ++ii_loc)
                {
                   const int ix = ii_loc%2;

fem/normal_deriv_restriction.cpp

@@ -1,828 +0,0 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "normal_deriv_restriction.hpp"
-#include "fespace.hpp"
-#include "pgridfunc.hpp"
-#include "fe/face_map_utils.hpp"
-#include "../general/forall.hpp"
-
-namespace mfem
-{
-
-/// Compute the face index to volume index map "face_to_vol" in 2D
-static void NormalDerivativeSetupFaceIndexMap2D(
-   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
-{
-   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 2, nf);
-   auto f2v = Reshape(face_to_vol.HostWrite(), d, 2, nf);
-
-   for (int f = 0; f < nf; ++f)
-   {
-      const int fid0 = f2e(0, 1, f);
-      const int fid1 = f2e(1, 1, f);
-      for (int side = 0; side < 2; ++side)
-      {
-         const int el = f2e(side, 0, f);
-
-         if (el < 0)
-         {
-            for (int p = 0; p < d; ++p)
-            {
-               f2v(p, side, f) = -1;
-            }
-         }
-         else
-         {
-            for (int p = 0; p < d; ++p)
-            {
-               int i, j;
-               internal::FaceIdxToVolIdx2D(p, d, fid0, fid1, side, i, j);
-
-               f2v(p, side, f) = i + d * j;
-            }
-         }
-      }
-   }
-}
-
-/// Compute the face index to volume index map "face_to_vol" in 3D
-static void NormalDerivativeSetupFaceIndexMap3D(
-   int nf, int d, const Array<int>& face_to_elem, Array<int>& face_to_vol)
-{
-   const auto f2e = Reshape(face_to_elem.HostRead(), 2, 3, nf);
-   auto f2v = Reshape(face_to_vol.HostWrite(), d*d, 2, nf);
-
-   for (int f = 0; f < nf; ++f)
-   {
-      const int fid0 = f2e(0, 1, f);
-      const int fid1 = f2e(1, 1, f);
-      for (int side = 0; side < 2; ++side)
-      {
-         const int el = f2e(side, 0, f);
-         const int orientation = f2e(side, 2, f);
-
-         if (el < 0)
-         {
-            for (int p = 0; p < d*d; ++p)
-            {
-               f2v(p, side, f) = -1;
-            }
-         }
-         else
-         {
-            for (int p = 0; p < d*d; ++p)
-            {
-               int i, j, k; // 3D lexicographic index of quad point p
-               internal::FaceIdxToVolIdx3D(p, d, fid0, fid1, side, orientation, i, j, k);
-
-               f2v(p, side, f) = i + d * (j + d * k);
-            }
-         }
-      }
-   }
-}
-
-L2NormalDerivativeFaceRestriction::L2NormalDerivativeFaceRestriction(
-   const FiniteElementSpace &fes_,
-   const ElementDofOrdering f_ordering,
-   const FaceType face_type_)
-   : fes(fes_),
-     face_type(face_type_),
-     dim(fes.GetMesh()->Dimension()),
-     nf(fes.GetNFbyType(face_type)),
-     ne(fes.GetNE())
-{
-   MFEM_VERIFY(f_ordering == ElementDofOrdering::LEXICOGRAPHIC,
-               "Non-lexicographic ordering not currently supported in "
-               "L2NormalDerivativeFaceRestriction.");
-
-   Mesh &mesh = *fes.GetMesh();
-
-   const FiniteElement &fe = *fes.GetFE(0);
-   const int d = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR).ndof;
-
-   if (dim == 2)
-   {
-      // (el0, el1, fid0, fid1)
-      face_to_elem.SetSize(nf * 4);
-      face_to_vol.SetSize(2 * nf * d);
-   }
-   else if (dim == 3)
-   {
-      // (el0, el1, fid0, fid1, or0, or1)
-      face_to_elem.SetSize(nf * 6);
-      face_to_vol.SetSize(2 * nf * d * d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension.");
-   }
-   auto f2e = Reshape(face_to_elem.HostWrite(), 2, (dim == 2) ? 2 : 3, nf);
-
-   // Populate the face_to_elem array. The elem_indicator will be used to count
-   // the number of elements that are adjacent to faces of the given type.
-   Array<int> elem_indicator(ne);
-   elem_indicator = 0;
-
-   int f_ind = 0;
-   for (int f = 0; f < fes.GetNF(); ++f)
-   {
-      Mesh::FaceInformation face = mesh.GetFaceInformation(f);
-
-      if (face.IsOfFaceType(face_type))
-      {
-         f2e(0, 0, f_ind) = face.element[0].index;
-         f2e(0, 1, f_ind) = face.element[0].local_face_id;
-         if (dim == 3)
-         {
-            f2e(0, 2, f_ind) = face.element[0].orientation;
-         }
-
-         elem_indicator[face.element[0].index] = 1;
-
-         if (face_type == FaceType::Interior)
-         {
-            const int el_idx_1 = face.element[1].index;
-            if (face.IsShared())
-            {
-               // Indicate shared face by index >= ne
-               f2e(1, 0, f_ind) = ne + el_idx_1;
-            }
-            else
-            {
-               // Face is not shared
-               f2e(1, 0, f_ind) = el_idx_1;
-               elem_indicator[el_idx_1] = 1;
-            }
-            f2e(1, 1, f_ind) = face.element[1].local_face_id;
-
-            if (dim == 3)
-            {
-               f2e(1, 2, f_ind) = face.element[1].orientation;
-            }
-         }
-         else
-         {
-            f2e(1, 0, f_ind) = -1;
-            f2e(1, 1, f_ind) = -1;
-
-            if (dim == 3)
-            {
-               f2e(1, 2, f_ind) = -1;
-            }
-         }
-
-         f_ind++;
-      }
-   }
-
-   // evaluate face to vol map
-   if (dim == 2)
-   {
-      NormalDerivativeSetupFaceIndexMap2D(nf, d, face_to_elem, face_to_vol);
-   }
-   else if (dim == 3)
-   {
-      NormalDerivativeSetupFaceIndexMap3D(nf, d, face_to_elem, face_to_vol);
-   }
-
-   // Number of elements adjacent to faces of face_type
-   ne_type = elem_indicator.Sum();
-
-   // In 2D: (el, f0,f1,f2,f3, s0,s1,s2,s3)
-   // In 3D: (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
-   const int elem_data_sz = (dim == 2) ? 9 : 13;
-
-   elem_to_face.SetSize(elem_data_sz * ne_type);
-   elem_to_face = -1;
-
-   auto e2f = Reshape(elem_to_face.HostWrite(), elem_data_sz, ne_type);
-   elem_indicator.PartialSum();
-
-   const int nsides = (face_type == FaceType::Interior) ? 2 : 1;
-   const int side_begin = (dim == 2) ? 5 : 7;
-   for (int f = 0; f < nf; ++f)
-   {
-      for (int side = 0; side < nsides; ++side)
-      {
-         const int el = f2e(side, 0, f);
-         // Skip shared faces
-         if (el < ne)
-         {
-            const int face_id = f2e(side, 1, f);
-
-            const int e = elem_indicator[el] - 1;
-            e2f(0, e) = el;
-            e2f(1 + face_id, e) = f;
-            e2f(side_begin + face_id, e) = side;
-         }
-      }
-   }
-}
-
-void L2NormalDerivativeFaceRestriction::Mult(const Vector &x, Vector &y) const
-{
-   if (nf == 0) { return; }
-   switch (dim)
-   {
-      case 2:
-      {
-         const int d1d = fes.GetElementOrder(0) + 1;
-         switch (d1d)
-         {
-            case 1: Mult2D<1>(x, y); break;
-            case 2: Mult2D<2>(x, y); break;
-            case 3: Mult2D<3>(x, y); break;
-            case 4: Mult2D<4>(x, y); break;
-            case 5: Mult2D<5>(x, y); break;
-            case 6: Mult2D<6>(x, y); break;
-            case 7: Mult2D<7>(x, y); break;
-            case 8: Mult2D<8>(x, y); break;
-            default: Mult2D(x, y); break;
-         }
-      }
-      break;
-      case 3:
-      {
-         const int d1d = fes.GetElementOrder(0) + 1;
-         switch (d1d)
-         {
-            case 1: Mult3D<1>(x, y); break;
-            case 2: Mult3D<2>(x, y); break;
-            case 3: Mult3D<3>(x, y); break;
-            case 4: Mult3D<4>(x, y); break;
-            case 5: Mult3D<5>(x, y); break;
-            case 6: Mult3D<6>(x, y); break;
-            case 7: Mult3D<7>(x, y); break;
-            case 8: Mult3D<8>(x, y); break;
-            default: Mult3D(x, y); break; // fallback
-         }
-         break;
-      }
-      default: MFEM_ABORT("Dimension not supported."); break;
-   }
-}
-
-void L2NormalDerivativeFaceRestriction::AddMultTranspose(
-   const Vector &x, Vector &y, const real_t a) const
-{
-   if (nf == 0) { return; }
-   switch (dim)
-   {
-      case 2:
-      {
-         const int d1d = fes.GetElementOrder(0) + 1;
-         switch (d1d)
-         {
-            case 1: AddMultTranspose2D<1>(x, y, a); break;
-            case 2: AddMultTranspose2D<2>(x, y, a); break;
-            case 3: AddMultTranspose2D<3>(x, y, a); break;
-            case 4: AddMultTranspose2D<4>(x, y, a); break;
-            case 5: AddMultTranspose2D<5>(x, y, a); break;
-            case 6: AddMultTranspose2D<6>(x, y, a); break;
-            case 7: AddMultTranspose2D<7>(x, y, a); break;
-            case 8: AddMultTranspose2D<8>(x, y, a); break;
-            default: AddMultTranspose2D(x, y, a); break;
-         }
-      }
-      break;
-      case 3:
-      {
-         const int d1d = fes.GetElementOrder(0) + 1;
-         switch (d1d)
-         {
-            case 1: AddMultTranspose3D<1>(x, y, a); break;
-            case 2: AddMultTranspose3D<2>(x, y, a); break;
-            case 3: AddMultTranspose3D<3>(x, y, a); break;
-            case 4: AddMultTranspose3D<4>(x, y, a); break;
-            case 5: AddMultTranspose3D<5>(x, y, a); break;
-            case 6: AddMultTranspose3D<6>(x, y, a); break;
-            case 7: AddMultTranspose3D<7>(x, y, a); break;
-            case 8: AddMultTranspose3D<8>(x, y, a); break;
-            default: AddMultTranspose3D(x, y, a); break; // fallback
-         }
-         break;
-      }
-      default: MFEM_ABORT("Not yet implemented"); break;
-   }
-}
-
-template <int T_D1D>
-void L2NormalDerivativeFaceRestriction::Mult2D(const Vector &x, Vector &y) const
-{
-   const int vd = fes.GetVDim();
-   const bool t = fes.GetOrdering() == Ordering::byVDIM;
-   const int num_elem = ne;
-
-   const FiniteElement &fe = *fes.GetFE(0);
-   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
-
-   const int q = maps.nqpt;
-   const int d = maps.ndof;
-
-   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
-   const int ne_shared = face_nbr_data.Size() / d / d / vd;
-
-   MFEM_VERIFY(q == d, "");
-   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
-
-   // derivative of 1D basis function
-   const auto G_ = Reshape(maps.G.Read(), q, d);
-   // (el0, el1, fid0, fid1)
-   const auto f2e = Reshape(face_to_elem.Read(), 2, 2, nf);
-
-   const auto f2v = Reshape(face_to_vol.Read(), q, 2, nf);
-
-   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
-   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
-   const auto d_x = Reshape(x.Read(), t?vd:d, d, t?d:ne, t?ne:vd);
-   const auto d_x_shared = Reshape(face_nbr_data.Read(),
-                                   t?vd:d, d, t?d:ne_shared, t?ne_shared:vd);
-   auto d_y = Reshape(y.Write(), q, vd, 2, nf);
-
-   mfem::forall_2D(nf, 2, q, [=] MFEM_HOST_DEVICE (int f) -> void
-   {
-      constexpr int MD = (T_D1D) ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED real_t G_s[MD*MD];
-      DeviceMatrix G(G_s, q, d);
-
-      MFEM_SHARED int E[2];
-      MFEM_SHARED int FID[2];
-      MFEM_SHARED int F2V[2][MD];
-
-      if (MFEM_THREAD_ID(x) == 0)
-      {
-         MFEM_FOREACH_THREAD(j, y, d)
-         {
-            for (int i = 0; i < q; ++i)
-            {
-               G(i, j) = G_(i, j);
-            }
-         }
-      }
-
-      MFEM_FOREACH_THREAD(side, x, 2)
-      {
-         if (MFEM_THREAD_ID(y) == 0)
-         {
-            E[side] = f2e(side, 0, f);
-            FID[side] = f2e(side, 1, f);
-         }
-
-         MFEM_FOREACH_THREAD(j, y, d)
-         {
-            F2V[side][j] = f2v(j, side, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, x, 2)
-      {
-         const int el = E[side];
-         const bool shared = (el >= num_elem);
-         const auto &d_x_e = shared ? d_x_shared : d_x;
-         const int el_idx = shared ? el - num_elem : el;
-
-         const int face_id = FID[side];
-
-         MFEM_FOREACH_THREAD(p, y, q)
-         {
-            if (el < 0)
-            {
-               for (int c = 0; c < vd; ++c)
-               {
-                  d_y(p, c, side, f) = 0.0;
-               }
-            }
-            else
-            {
-               const int ij = F2V[side][p];
-               const int i = ij % q;
-               const int j = ij / q;
-
-               for (int c=0; c < vd; ++c)
-               {
-                  real_t grad_n = 0;
-                  for (int kk=0; kk < d; ++kk)
-                  {
-                     const int k = (face_id == 0 || face_id == 2) ? i : kk;
-                     const int l = (face_id == 0 || face_id == 2) ? kk : j;
-                     const real_t g = (face_id == 0 || face_id == 2) ? G(j,l) : G(i,k);
-                     grad_n += g * d_x_e(t?c:k, t?k:l, t?l:el_idx, t?el_idx:c);
-                  }
-                  d_y(p, c, side, f) = grad_n;
-               }
-            }
-         }
-      }
-   });
-}
-
-template <int T_D1D>
-void L2NormalDerivativeFaceRestriction::Mult3D(const Vector &x, Vector &y) const
-{
-   const int vd = fes.GetVDim();
-   const bool t = fes.GetOrdering() == Ordering::byVDIM;
-   const int num_elem = ne;
-
-   const FiniteElement &fe = *fes.GetFE(0);
-   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
-
-   const int q = maps.nqpt;
-   const int d = maps.ndof;
-   const int q2d = q * q;
-
-   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, face_type);
-   const int ne_shared = face_nbr_data.Size() / d / d / d / vd;
-
-   MFEM_VERIFY(q == d, "");
-   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
-
-   const auto G_ = Reshape(maps.G.Read(), q, d);
-   // (el0, el1, fid0, fid1, or0, or1)
-   const auto f2e = Reshape(face_to_elem.Read(), 2, 3, nf);
-   const auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
-
-   // t ? (vdim, d, d, d, ne) : (d, d, d, ne, vdim)
-   const auto d_x = Reshape(x.Read(), t?vd:d, d, d, t?d:ne, t?ne:vd);
-   const auto d_x_shared = Reshape(face_nbr_data.Read(),
-                                   t?vd:d, d, d, t?d:ne_shared, t?ne_shared:vd);
-   auto d_y = Reshape(y.Write(), q2d, vd, 2, nf);
-
-   mfem::forall_2D(nf, q2d, 2, [=] MFEM_HOST_DEVICE (int f) -> void
-   {
-      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED real_t G_s[MD*MD];
-      DeviceMatrix G(G_s, d, q);
-
-      MFEM_SHARED int E[2];
-      MFEM_SHARED int FID[2];
-      MFEM_SHARED int F2V[2][MD*MD];
-
-      // Load G matrix into shared memory
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         MFEM_FOREACH_THREAD(j, x, d*q)
-         {
-            const int p = j % q;
-            const int k = j / q;
-            G(k, p) = G_(p, k);
-         }
-      }
-
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         if (MFEM_THREAD_ID(x) == 0)
-         {
-            E[side] = f2e(side, 0, f);
-            FID[side] = f2e(side, 1, f);
-         }
-         MFEM_FOREACH_THREAD(j, x, q2d)
-         {
-            F2V[side][j] = f2v(j, side, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         const int el = E[side];
-         const bool shared = (el >= num_elem);
-         const auto &d_x_e = shared ? d_x_shared : d_x;
-         const int el_idx = shared ? el - num_elem : el;
-
-         const int face_id = FID[side];
-
-         // Is this face parallel to the x-y plane in reference coordinates?
-         const bool xy_plane = (face_id == 0 || face_id == 5);
-         const bool xz_plane = (face_id == 1 || face_id == 3);
-         const bool yz_plane = (face_id == 2 || face_id == 4);
-
-         MFEM_FOREACH_THREAD(p, x, q2d)
-         {
-            if (el_idx < 0)
-            {
-               for (int c = 0; c < vd; ++c)
-               {
-                  d_y(p, c, side, f) = 0.0;
-               }
-            }
-            else
-            {
-               const int ijk = F2V[side][p];
-               const int k = ijk / q2d;
-               const int i = ijk % q;
-               const int j = (ijk - q2d*k) / q;
-
-               // the fixed 1D index of the normal component of the face
-               // quadrature point
-               const int g_row = yz_plane ? i : xz_plane ? j : k;
-
-               for (int c = 0; c < vd; ++c)
-               {
-                  real_t grad_n = 0.0;
-
-                  for (int kk = 0; kk < d; ++kk)
-                  {
-                     // (l, m, n) 3D lexicographic index of interior points used
-                     // in evaluating normal derivatives
-                     const int l = yz_plane ? kk : i;
-                     const int m = xz_plane ? kk : j;
-                     const int n = xy_plane ? kk : k;
-
-                     const real_t g = G(kk, g_row);
-
-                     grad_n += g * d_x_e(t?c:l, t?l:m, t?m:n, t?n:el_idx, t?el_idx:c);
-                  }
-                  d_y(p, c, side, f) = grad_n;
-               }
-            }
-         }
-      }
-   });
-}
-
-template <int T_D1D>
-void L2NormalDerivativeFaceRestriction::AddMultTranspose2D(
-   const Vector &y, Vector &x, const real_t a) const
-{
-   const int vd = fes.GetVDim();
-   const bool t = fes.GetOrdering() == Ordering::byVDIM;
-
-   const FiniteElement &fe = *fes.GetFE(0);
-   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
-
-   const int q = maps.nqpt;
-   const int d = maps.ndof;
-
-   // derivative of 1D basis function
-   auto G_ = Reshape(maps.G.Read(), q, d);
-
-   // entries of e2f: (el,f0,f1,f2,f3,s0,s1,s2,s3)
-   auto e2f = Reshape(elem_to_face.Read(), 9, ne_type);
-
-   auto f2v = Reshape(face_to_vol.Read(), d, 2, nf);
-
-   // if byvdim, d_x has shape (vdim, nddof, nddof, ne)
-   // otherwise, d_x has shape (nddof, nddof, ne, vdim)
-   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, t?d:ne, t?ne:vd);
-   auto d_y = Reshape(y.Read(), q, vd, 2, nf);
-
-   mfem::forall_2D(ne_type, d, d, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED real_t y_s[MD];
-      MFEM_SHARED int pp[MD];
-      MFEM_SHARED int jj;
-      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
-
-      MFEM_SHARED real_t BG[MD*MD];
-      DeviceMatrix G(BG, q, d);
-
-      MFEM_SHARED real_t x_s[MD*MD];
-      DeviceMatrix xx(x_s, d, d);
-
-      MFEM_SHARED int el; // global element index
-      MFEM_SHARED int faces[4];
-      MFEM_SHARED int sides[4];
-
-      MFEM_FOREACH_THREAD(i,x,d)
-      {
-         MFEM_FOREACH_THREAD(p,y,q)
-         {
-            G(p,i) = a * G_(p,i);
-            xx(p,i) = 0.0;
-         }
-      }
-
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         if (MFEM_THREAD_ID(x) == 0)
-         {
-            el = e2f(0, e);
-         }
-
-         MFEM_FOREACH_THREAD(i, x, 4)
-         {
-            faces[i] = e2f(1 + i, e);
-            sides[i] = e2f(5 + i, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int face_id=0; face_id < 4; ++face_id)
-      {
-         const int f = faces[face_id];
-
-         if (f < 0) { continue; }
-
-         const int side = sides[face_id];
-
-         if (MFEM_THREAD_ID(y) == 0)
-         {
-            MFEM_FOREACH_THREAD(p,x,d)
-            {
-               y_s[p] = d_y(p, 0, side, f);
-
-               const int ij = f2v(p, side, f);
-               const int i = ij % q;
-               const int j = ij / q;
-
-               pp[(face_id == 0 || face_id == 2) ? i : j] = p;
-               if (MFEM_THREAD_ID(x) == 0)
-               {
-                  jj = (face_id == 0 || face_id == 2) ? j : i;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(k,x,d)
-         {
-            MFEM_FOREACH_THREAD(l,y,d)
-            {
-               const int p = (face_id == 0 || face_id == 2) ? pp[k] : pp[l];
-               const int kk = (face_id == 0 || face_id == 2) ? l : k;
-               const real_t g = G(jj, kk);
-               xx(k,l) += g * y_s[p];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(k,x,d)
-      {
-         MFEM_FOREACH_THREAD(l,y,d)
-         {
-            const int c = 0;
-            d_x(t?c:k, t?k:l, t?l:el, t?el:c) += xx(k,l);
-         }
-      }
-   });
-}
-
-template <int T_D1D>
-void L2NormalDerivativeFaceRestriction::AddMultTranspose3D(
-   const Vector &y, Vector &x, const real_t a) const
-{
-   const int vd = fes.GetVDim();
-   const bool t = fes.GetOrdering() == Ordering::byVDIM;
-
-   MFEM_VERIFY(vd == 1, "vdim > 1 not supported.");
-
-   const FiniteElement &fe = *fes.GetFE(0);
-   const DofToQuad &maps = fe.GetDofToQuad(fe.GetNodes(), DofToQuad::TENSOR);
-
-   const int q = maps.nqpt;
-   const int d = maps.ndof;
-   const int q2d = q * q;
-
-   MFEM_VERIFY(q == d, "");
-   MFEM_VERIFY(T_D1D == d || T_D1D == 0, "");
-
-   auto G_ = Reshape(maps.G.Read(), q, d);
-
-   // (el, f0,f1,f2,f3,f4,f5, s0,s1,s2,s3,s4,s5)
-   auto e2f = Reshape(elem_to_face.Read(), 13, ne_type);
-
-   auto f2v = Reshape(face_to_vol.Read(), q2d, 2, nf);
-
-   auto d_x = Reshape(x.ReadWrite(), t?vd:d, d, d, t?d:ne, t?ne:vd);
-   const auto d_y = Reshape(y.Read(), q2d, vd, 2, nf);
-
-   mfem::forall_2D(ne_type, q, q, [=] MFEM_HOST_DEVICE (int e) -> void
-   {
-      static constexpr int MD = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED int pp[MD][MD];
-      MFEM_SHARED real_t y_s[MD*MD];
-      MFEM_SHARED int jj;
-      if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0) { jj = 0; }
-
-      MFEM_SHARED real_t xx_s[MD*MD*MD];
-      auto xx = Reshape(xx_s, d, d, d);
-
-      MFEM_SHARED real_t G_s[MD*MD];
-      DeviceMatrix G(G_s, q, d);
-
-      MFEM_SHARED int el;
-      MFEM_SHARED int faces[6];
-      MFEM_SHARED int sides[6];
-
-      // Load G into shared memory
-      MFEM_FOREACH_THREAD(j, x, d)
-      {
-         MFEM_FOREACH_THREAD(i, y, q)
-         {
-            G(i, j) = a * G_(i, j);
-            G(i, j) = a * G_(i, j);
-            G(i, j) = a * G_(i, j);
-         }
-      }
-
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         if (MFEM_THREAD_ID(x) == 0)
-         {
-            el = e2f(0, e); // global element index
-         }
-
-         MFEM_FOREACH_THREAD(i, x, 6)
-         {
-            faces[i] = e2f(1 + i, e);
-            sides[i] = e2f(7 + i, e);
-         }
-      }
-
-      MFEM_FOREACH_THREAD(k, x, d)
-      {
-         MFEM_FOREACH_THREAD(j, y, d)
-         {
-            for (int i = 0; i < d; ++i)
-            {
-               xx(i, j, k) = 0.0;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      for (int face_id = 0; face_id < 6; ++face_id)
-      {
-         const int f = faces[face_id];
-
-         if (f < 0)
-         {
-            continue;
-         }
-
-         const int side = sides[face_id];
-
-         // is this face parallel to the x-y plane in reference coordinates?
-         const bool xy_plane = (face_id == 0 || face_id == 5);
-         const bool xz_plane = (face_id == 1 || face_id == 3);
-
-         MFEM_FOREACH_THREAD(p1, x, q)
-         {
-            MFEM_FOREACH_THREAD(p2, y, q)
-            {
-               const int p = p1 + q * p2;
-               y_s[p] = d_y(p, 0, side, f);
-
-               const int ijk = f2v(p, side, f);
-               const int k = ijk / q2d;
-               const int i = ijk % q;
-               const int j = (ijk - q2d*k) / q;
-
-               pp[(xy_plane || xz_plane) ? i : j][(xy_plane) ? j : k] = p;
-               if (MFEM_THREAD_ID(x) == 0 && MFEM_THREAD_ID(y) == 0)
-               {
-                  jj = (xy_plane) ? k : (xz_plane) ? j : i;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(n, x, d)
-         {
-            MFEM_FOREACH_THREAD(m, y, d)
-            {
-               for (int l = 0; l < d; ++l)
-               {
-                  const int p = (xy_plane) ? pp[l][m] : (xz_plane) ? pp[l][n] : pp[m][n];
-                  const int kk = (xy_plane) ? n : (xz_plane) ? m : l;
-                  const real_t g = G(jj, kk);
-                  xx(l, m, n) += g * y_s[p];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // map back to global array
-      MFEM_FOREACH_THREAD(n, x, d)
-      {
-         MFEM_FOREACH_THREAD(m, y, d)
-         {
-            for (int l = 0; l < d; ++l)
-            {
-               const int c = 0;
-               d_x(t?c:l, t?l:m, t?m:n, t?n:el, t?el:c) += xx(l, m, n);
-            }
-         }
-      }
-   });
-}
-
-} // namespace mfem

fem/normal_deriv_restriction.hpp

@@ -1,85 +0,0 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_NORMAL_DERIV_RESTRICTION
-#define MFEM_NORMAL_DERIV_RESTRICTION
-
-#include "../mesh/mesh.hpp"
-
-namespace mfem
-{
-
-class FiniteElementSpace;
-enum class ElementDofOrdering;
-
-/// @brief Class to compute face normal derivatives (in reference coordinate) of
-/// an L2 grid function (used internally by L2FaceRestriction).
-class L2NormalDerivativeFaceRestriction
-{
-protected:
-   const FiniteElementSpace &fes; ///< The L2 finite element space.
-   const FaceType face_type; ///< Face type: either boundary or interior.
-   const int dim; ///< Dimension of the mesh.
-   const int nf; ///< Number of faces of the given @a face_type.
-   const int ne; ///< Number of elements.
-   int ne_type; ///< Number of elements with faces of type face type
-
-   Array<int> face_to_elem; ///< Face-wise information array.
-   Array<int> elem_to_face; ///< Element-wise information array.
-   Array<int> face_to_vol; ///< maps face index to volume index
-
-public:
-   /// @brief Constructor.
-   /// @param[in] fes_ The associated FiniteElementSpace (should be L2/DG).
-   /// @param[in] f_ordering Request a specific face dof ordering. Currently
-   /// only ElementDofOrdering::LEXICOGRAPHIC is supported.
-   /// @param[in] face_type_ Type of faces to compute restriction (interior or boundary).
-   L2NormalDerivativeFaceRestriction(const FiniteElementSpace &fes_,
-                                     const ElementDofOrdering f_ordering,
-                                     const FaceType face_type_);
-
-   /// @brief Computes the normal derivatives on the @a face_type faces of the mesh.
-   /// @param[in] x The L-vector degrees of freedom.
-   /// @param[out] y The face E(like)-vector degrees of freedom of the format
-   /// (face_dofs x vdim x 2 x nf) where nf is the number of faces of type @a
-   /// face_type. The face_dofs are ordered according to @a f_ordering specified
-   /// in the constructor.
-   void Mult(const Vector &x, Vector &y) const;
-
-   /// @brief Computes the transpose of the action of Mult(), accumulating into
-   ///        @a y with coefficient @a a.
-   /// @param x Face E-vector layout (face_dofs x vdim x 2 x nf).
-   /// @param y L-vector layout.
-   /// @param a Optional coefficient (y = y + a*R^t*x)
-   void AddMultTranspose(const Vector &x, Vector &y,
-                         const real_t a = 1.0) const;
-
-   /// @name Internal compute kernels. Public because of nvcc restriction.
-   ///@{
-
-   template <int T_D1D = 0>
-   void Mult2D(const Vector &x, Vector &y) const;
-
-   template <int T_D1D = 0>
-   void AddMultTranspose2D(const Vector &x, Vector &y, const real_t a) const;
-
-   template <int T_D1D = 0>
-   void Mult3D(const Vector &x, Vector &y) const;
-
-   template <int T_D1D = 0>
-   void AddMultTranspose3D(const Vector &x, Vector &y, const real_t a) const;
-
-   /// @}
-};
-
-}
-
-#endif // MFEM_RESTRICTION

fem/pbilinearform.cpp

@@ -368,72 +368,6 @@ const
    y.Add(a, Ytmp);
 }
 
-real_t ParBilinearForm::ParInnerProduct(const ParGridFunction &x,
-                                        const ParGridFunction &y) const
-{
-   MFEM_ASSERT(mat != NULL, "local matrix must be assembled");
-
-   real_t loc = InnerProduct(x, y);
-   real_t glob = 0.;
-
-   MPI_Allreduce(&loc, &glob, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
-                 pfes->GetComm());
-
-   return glob;
-}
-
-real_t ParBilinearForm::TrueInnerProduct(const ParGridFunction &x,
-                                         const ParGridFunction &y) const
-{
-   MFEM_ASSERT(x.ParFESpace() == pfes, "the parallel spaces must match");
-   MFEM_ASSERT(y.ParFESpace() == pfes, "the parallel spaces must match");
-
-   HypreParVector *x_p = x.ParallelProject();
-   HypreParVector *y_p = y.ParallelProject();
-
-   real_t res = TrueInnerProduct(*x_p, *y_p);
-
-   delete x_p;
-   delete y_p;
-
-   return res;
-}
-
-real_t ParBilinearForm::TrueInnerProduct(HypreParVector &x,
-                                         HypreParVector &y) const
-{
-   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
-
-   if (p_mat->GetType() != Operator::Hypre_ParCSR)
-   {
-      return TrueInnerProduct((const Vector&)x, (const Vector&)y);
-   }
-
-   HypreParVector *Ax = new HypreParVector(pfes);
-   HypreParMatrix *A = p_mat.As<HypreParMatrix>();
-
-   A->Mult(x, *Ax);
-
-   real_t res = mfem::InnerProduct(y, *Ax);
-
-   delete Ax;
-
-   return res;
-}
-
-real_t ParBilinearForm::TrueInnerProduct(const Vector &x,
-                                         const Vector &y) const
-{
-   MFEM_VERIFY(p_mat.Ptr() != NULL, "parallel matrix must be assembled");
-
-   Vector Ax(pfes->GetTrueVSize());
-   p_mat->Mult(x, Ax);
-
-   real_t res = mfem::InnerProduct(pfes->GetComm(), y, Ax);
-
-   return res;
-}
-
 void ParBilinearForm::FormLinearSystem(
    const Array<int> &ess_tdof_list, Vector &x, Vector &b,
    OperatorHandle &A, Vector &X, Vector &B, int copy_interior)

fem/pbilinearform.hpp

@@ -173,37 +173,6 @@ public:
        vectors on the true dofs. */
    void TrueAddMult(const Vector &x, Vector &y, const real_t a = 1.0) const;
 
-   /// Compute $ y^T M x $
-   /** @warning The calculation is performed on local dofs, assuming that
-       the local vectors are consistent with the prolongations of the true
-       vectors (see ParGridFunction::Distribute()). If this is not the case,
-       use TrueInnerProduct(const ParGridFunction &, const ParGridFunction &)
-       instead.
-       @note It is assumed that the local matrix is assembled and it has
-       not been replaced by the parallel matrix through FormSystemMatrix().
-       @see TrueInnerProduct(const ParGridFunction&, const ParGridFunction&) */
-   real_t ParInnerProduct(const ParGridFunction &x,
-                          const ParGridFunction &y) const;
-
-   /// Compute $ y^T M x $ on true dofs (grid function version)
-   /** @note The ParGridFunction%s are restricted to the true-vectors for
-       for calculation.
-       @note It is assumed that the parallel system matrix is assembled,
-       see FormSystemMatrix().
-       @see ParInnerProduct(const ParGridFunction&, const ParGridFunction&) */
-   real_t TrueInnerProduct(const ParGridFunction &x,
-                           const ParGridFunction &y) const;
-
-   /// Compute $ y^T M x $ on true dofs (Hypre vector version)
-   /** @note It is assumed that the parallel system matrix is assembled,
-       see FormSystemMatrix(). */
-   real_t TrueInnerProduct(HypreParVector &x, HypreParVector &y) const;
-
-   /// Compute $ y^T M x $ on true dofs (true-vector version)
-   /** @note It is assumed that the parallel system matrix is assembled,
-       see FormSystemMatrix(). */
-   real_t TrueInnerProduct(const Vector &x, const Vector &y) const;
-
    /// Return the parallel FE space associated with the ParBilinearForm.
    ParFiniteElementSpace *ParFESpace() const { return pfes; }
 

fem/pfespace.cpp

@@ -861,17 +861,17 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
          }
       }
 
-      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
-      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
-      real_t    *d_diag = Memory<real_t>(ltdof);
+      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
+      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
+      real_t    *d_diag = new real_t[ltdof];
       int diag_counter;
 
-      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
-      HYPRE_Int *j_offd = Memory<HYPRE_Int>(nnz_offd);
-      real_t    *d_offd = Memory<real_t>(nnz_offd);
+      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
+      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
+      real_t    *d_offd = new real_t[nnz_offd];
       int offd_counter;
 
-      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);
+      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
 
       HYPRE_BigInt *col_starts = GetTrueDofOffsets();
       HYPRE_BigInt *row_starts = GetDofOffsets();

fem/pgridfunc.cpp

@@ -249,8 +249,6 @@ void ParGridFunction::ExchangeFaceNbrData()
    auto send_data_ptr = mpi_gpu_aware ? send_data.Read() : send_data.HostRead();
    auto face_nbr_data_ptr = mpi_gpu_aware ? face_nbr_data.Write() :
                             face_nbr_data.HostWrite();
-   // Wait for the kernel to be done since it updates what's sent and it may be async
-   if (mpi_gpu_aware) { MFEM_STREAM_SYNC; }
    for (int fn = 0; fn < num_face_nbrs; fn++)
    {
       int nbr_rank = pmesh->GetFaceNbrRank(fn);
@@ -520,7 +518,7 @@ void ParGridFunction::CountElementsPerVDof(Array<int> &elem_per_vdof) const
 }
 
 void ParGridFunction::GetDerivative(int comp, int der_comp,
-                                    ParGridFunction &der) const
+                                    ParGridFunction &der)
 {
    Array<int> overlap;
    AccumulateAndCountDerivativeValues(comp, der_comp, der, overlap);
@@ -715,10 +713,10 @@ void ParGridFunction::ProjectBdrCoefficient(
          }
       }
    }
-   gcomm.Bcast<int>(values_counter.HostReadWrite());
    for (int i = 0; i < values_counter.Size(); i++)
    {
-      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
+                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                   "internal error");
    }
 #endif
@@ -755,10 +753,10 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 #ifdef MFEM_DEBUG
    Array<int> ess_vdofs_marker;
    pfes->GetEssentialVDofs(bdr_attr, ess_vdofs_marker);
-   gcomm.Bcast<int>(values_counter.HostReadWrite());
    for (int i = 0; i < values_counter.Size(); i++)
    {
-      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(pfes->GetLocalTDofNumber(i) == -1 ||
+                  bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                   "internal error: " << pfes->GetLocalTDofNumber(i) << ' ' << bool(
                      values_counter[i]));
    }

fem/pgridfunc.hpp

@@ -231,7 +231,7 @@ public:
    void CountElementsPerVDof(Array<int> &elem_per_vdof) const override;
 
    /// Parallel version of GridFunction::GetDerivative(); see its documentation.
-   void GetDerivative(int comp, int der_comp, ParGridFunction &der) const;
+   void GetDerivative(int comp, int der_comp, ParGridFunction &der);
 
    /** Sets the output vector @a dof_vals to the values of the degrees of
        freedom of element @a el. If @a el is greater than or equal to the number
@@ -262,6 +262,7 @@ public:
                               const Array<int> &attr) override
    { ProjectBdrCoefficient(coeff, NULL, attr); }
 
+   // Only the values in the master are guaranteed to be correct!
    void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                      const Array<int> &bdr_attr) override;
 

fem/prestriction.cpp

@@ -18,7 +18,6 @@
 #include "pgridfunc.hpp"
 #include "pfespace.hpp"
 #include "fespace.hpp"
-#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"
 
 namespace mfem
@@ -278,22 +277,21 @@ void ParNCH1FaceRestriction::ComputeGatherIndices(
    gather_offsets[0] = 0;
 }
 
-ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
+ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
                                            ElementDofOrdering f_ordering,
                                            FaceType type,
                                            L2FaceValues m,
                                            bool build)
-   : L2FaceRestriction(pfes_, f_ordering, type, m, false),
-     pfes(pfes_)
+   : L2FaceRestriction(fes, f_ordering, type, m, false)
 {
    if (!build) { return; }
    if (nf==0) { return; }
 
-   CheckFESpace();
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets();
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
 
-   ComputeGatherIndices();
+   ComputeGatherIndices(f_ordering, type);
 }
 
 ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
@@ -309,8 +307,14 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
    MFEM_ASSERT(
       m == L2FaceValues::DoubleValued,
       "This method should be called when m == L2FaceValues::DoubleValued.");
-
-   Vector face_nbr_data = GetLVectorFaceNbrData(fes, x, type);
+   const ParFiniteElementSpace &pfes =
+      static_cast<const ParFiniteElementSpace&>(this->fes);
+   ParGridFunction x_gf;
+   x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(&pfes),
+                const_cast<Vector&>(x), 0);
+   // Face-neighbor information is only needed for interior faces. For boundary
+   // faces, no communication is required.
+   if (type == FaceType::Interior) { x_gf.ExchangeFaceNbrData(); }
 
    // Early return only after calling ParGridFunction::ExchangeFaceNbrData,
    // otherwise MPI communication can hang.
@@ -325,7 +329,7 @@ void ParL2FaceRestriction::DoubleValuedConformingMult(
    auto d_indices1 = scatter_indices1.Read();
    auto d_indices2 = scatter_indices2.Read();
    auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_x_shared = Reshape(face_nbr_data.Read(),
+   auto d_x_shared = Reshape(x_gf.FaceNbrData().Read(),
                              t?vd:nsdofs, t?nsdofs:vd);
    auto d_y = Reshape(y.Write(), nface_dofs, vd, 2, nf);
    mfem::forall(nfdofs, [=] MFEM_HOST_DEVICE (int i)
@@ -563,9 +567,13 @@ void ParL2FaceRestriction::FillJAndData(const Vector &ea_data,
    });
 }
 
-void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets()
+void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets(
+   const ElementDofOrdering f_ordering,
+   const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
+   const ParFiniteElementSpace &pfes =
+      static_cast<const ParFiniteElementSpace&>(this->fes);
 
    // Initialization of the offsets
    for (int i = 0; i <= ndofs; ++i)
@@ -614,7 +622,9 @@ void ParL2FaceRestriction::ComputeScatterIndicesAndOffsets()
 }
 
 
-void ParL2FaceRestriction::ComputeGatherIndices()
+void ParL2FaceRestriction::ComputeGatherIndices(
+   const ElementDofOrdering f_ordering,
+   const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
 
@@ -656,11 +666,11 @@ ParNCL2FaceRestriction::ParNCL2FaceRestriction(const ParFiniteElementSpace &fes,
    if (nf==0) { return; }
    x_interp.UseDevice(true);
 
-   CheckFESpace();
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets();
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
 
-   ComputeGatherIndices();
+   ComputeGatherIndices(f_ordering, type);
 }
 
 void ParNCL2FaceRestriction::SingleValuedNonconformingMult(
@@ -969,7 +979,9 @@ void ParNCL2FaceRestriction::FillJAndData(const Vector &ea_data,
    MFEM_ABORT("Not yet implemented.");
 }
 
-void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
+void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
+   const ElementDofOrdering f_ordering,
+   const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
 
@@ -1052,7 +1064,9 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
    interpolations.InitializeNCInterpConfig();
 }
 
-void ParNCL2FaceRestriction::ComputeGatherIndices()
+void ParNCL2FaceRestriction::ComputeGatherIndices(
+   const ElementDofOrdering f_ordering,
+   const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
 

fem/prestriction.hpp

@@ -139,11 +139,9 @@ public: // For nvcc
 class ParL2FaceRestriction : virtual public L2FaceRestriction
 {
 protected:
-   const ParFiniteElementSpace &pfes;
-
    /** @brief Constructs an ParL2FaceRestriction.
 
-       @param[in] pfes_      The ParFiniteElementSpace on which this operates
+       @param[in] fes        The ParFiniteElementSpace on which this operates
        @param[in] f_ordering Request a specific face dof ordering
        @param[in] type       Request internal or boundary faces dofs
        @param[in] m          Request the face dofs for elem1, or both elem1 and
@@ -151,7 +149,7 @@ protected:
        @param[in] build      Request the ParL2FaceRestriction to compute the
                              scatter/gather indices. False should only be used
                              when inheriting from ParL2FaceRestriction. */
-   ParL2FaceRestriction(const ParFiniteElementSpace &pfes_,
+   ParL2FaceRestriction(const ParFiniteElementSpace& fes,
                         ElementDofOrdering f_ordering,
                         FaceType type,
                         L2FaceValues m,
@@ -231,14 +229,22 @@ public:
 private:
    /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
        for the gathering: E-vector to L-vector.
+
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets();
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
+                                        const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
+
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices();
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
+                             const FaceType type);
 
 public:
    /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -375,14 +381,22 @@ private:
    /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
        for the gathering: E-vector to L-vector, and the interpolators from
        coarse to fine face for master non-comforming faces.
+
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets();
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
+                                        const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
+
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices();
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
+                             const FaceType type);
 
 public:
    /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to

fem/qspace.hpp

@@ -215,10 +215,6 @@ public:
    /// quadrature point, oriented relative to "element 1".
    int GetPermutedIndex(int idx, int iq) const override;
 
-   /// @brief Get the face index (in the standard Mesh numbering) associated
-   /// with face @a idx in the FaceQuadratureSpace.
-   int GetMeshFaceIndex(int idx) const { return face_indices[idx]; }
-
    /// @brief Returns the index associated with the face described by @a T.
    ///
    /// The index may differ from the mesh face or boundary element index

fem/restriction.cpp

@@ -10,16 +10,17 @@
 // CONTRIBUTING.md for details.
 
 #include "restriction.hpp"
-#include "normal_deriv_restriction.hpp"
 #include "gridfunc.hpp"
 #include "fespace.hpp"
-#include "pgridfunc.hpp"
-#include "qspace.hpp"
-#include "fe/face_map_utils.hpp"
 #include "../general/forall.hpp"
-
 #include <climits>
 
+#ifdef MFEM_USE_MPI
+
+#include "pfespace.hpp"
+
+#endif
+
 namespace mfem
 {
 
@@ -906,6 +907,112 @@ void ConformingFaceRestriction::SetFaceDofsGatherIndices(
    }
 }
 
+static int ToLexOrdering2D(const int face_id, const int size1d, const int i)
+{
+   if (face_id==2 || face_id==3)
+   {
+      return size1d-1-i;
+   }
+   else
+   {
+      return i;
+   }
+}
+
+static int PermuteFace2D(const int face_id1, const int face_id2,
+                         const int orientation,
+                         const int size1d, const int index)
+{
+   int new_index;
+   // Convert from lex ordering
+   if (face_id1==2 || face_id1==3)
+   {
+      new_index = size1d-1-index;
+   }
+   else
+   {
+      new_index = index;
+   }
+   // Permute based on face orientations
+   if (orientation==1)
+   {
+      new_index = size1d-1-new_index;
+   }
+   return ToLexOrdering2D(face_id2, size1d, new_index);
+}
+
+static int ToLexOrdering3D(const int face_id, const int size1d, const int i,
+                           const int j)
+{
+   if (face_id==2 || face_id==1 || face_id==5)
+   {
+      return i + j*size1d;
+   }
+   else if (face_id==3 || face_id==4)
+   {
+      return (size1d-1-i) + j*size1d;
+   }
+   else // face_id==0
+   {
+      return i + (size1d-1-j)*size1d;
+   }
+}
+
+static int PermuteFace3D(const int face_id1, const int face_id2,
+                         const int orientation,
+                         const int size1d, const int index)
+{
+   int i=0, j=0, new_i=0, new_j=0;
+   i = index%size1d;
+   j = index/size1d;
+   // Convert from lex ordering
+   if (face_id1==3 || face_id1==4)
+   {
+      i = size1d-1-i;
+   }
+   else if (face_id1==0)
+   {
+      j = size1d-1-j;
+   }
+   // Permute based on face orientations
+   switch (orientation)
+   {
+      case 0:
+         new_i = i;
+         new_j = j;
+         break;
+      case 1:
+         new_i = j;
+         new_j = i;
+         break;
+      case 2:
+         new_i = j;
+         new_j = (size1d-1-i);
+         break;
+      case 3:
+         new_i = (size1d-1-i);
+         new_j = j;
+         break;
+      case 4:
+         new_i = (size1d-1-i);
+         new_j = (size1d-1-j);
+         break;
+      case 5:
+         new_i = (size1d-1-j);
+         new_j = (size1d-1-i);
+         break;
+      case 6:
+         new_i = (size1d-1-j);
+         new_j = i;
+         break;
+      case 7:
+         new_i = i;
+         new_j = (size1d-1-j);
+         break;
+   }
+   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
+}
+
 // Permute dofs or quads on a face for e2 to match with the ordering of e1
 int PermuteFaceL2(const int dim, const int face_id1,
                   const int face_id2, const int orientation,
@@ -916,9 +1023,9 @@ int PermuteFaceL2(const int dim, const int face_id1,
       case 1:
          return 0;
       case 2:
-         return internal::PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
+         return PermuteFace2D(face_id1, face_id2, orientation, size1d, index);
       case 3:
-         return internal::PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
+         return PermuteFace3D(face_id1, face_id2, orientation, size1d, index);
       default:
          MFEM_ABORT("Unsupported dimension.");
          return 0;
@@ -931,7 +1038,6 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
                                      const L2FaceValues m,
                                      bool build)
    : fes(fes),
-     ordering(f_ordering),
      nf(fes.GetNFbyType(type)),
      ne(fes.GetNE()),
      vdim(fes.GetVDim()),
@@ -954,9 +1060,11 @@ L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
    width = fes.GetVSize();
    if (!build) { return; }
 
-   CheckFESpace();
-   ComputeScatterIndicesAndOffsets();
-   ComputeGatherIndices();
+   CheckFESpace(f_ordering);
+
+   ComputeScatterIndicesAndOffsets(f_ordering,type);
+
+   ComputeGatherIndices(f_ordering, type);
 }
 
 L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
@@ -1219,7 +1327,7 @@ void L2FaceRestriction::AddFaceMatricesToElementMatrices(const Vector &fea_data,
    }
 }
 
-void L2FaceRestriction::CheckFESpace()
+void L2FaceRestriction::CheckFESpace(const ElementDofOrdering f_ordering)
 {
 #ifdef MFEM_USE_MPI
 
@@ -1243,7 +1351,7 @@ void L2FaceRestriction::CheckFESpace()
                "Only Gauss-Lobatto and Bernstein basis are supported in "
                "L2FaceRestriction.");
    if (nf==0) { return; }
-   const bool dof_reorder = (ordering == ElementDofOrdering::LEXICOGRAPHIC);
+   const bool dof_reorder = (f_ordering == ElementDofOrdering::LEXICOGRAPHIC);
    if (!dof_reorder)
    {
       MFEM_ABORT("Non-Tensor L2FaceRestriction not yet implemented.");
@@ -1263,7 +1371,9 @@ void L2FaceRestriction::CheckFESpace()
 #endif
 }
 
-void L2FaceRestriction::ComputeScatterIndicesAndOffsets()
+void L2FaceRestriction::ComputeScatterIndicesAndOffsets(
+   const ElementDofOrdering f_ordering,
+   const FaceType face_type)
 {
    Mesh &mesh = *fes.GetMesh();
    // Initialization of the offsets
@@ -1279,16 +1389,16 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets()
       Mesh::FaceInformation face = mesh.GetFaceInformation(f);
       MFEM_ASSERT(!face.IsShared(),
                   "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(type) )
+      if ( face.IsOfFaceType(face_type) )
       {
          SetFaceDofsScatterIndices1(face,f_ind);
          if ( m==L2FaceValues::DoubleValued )
          {
-            if ( type==FaceType::Interior && face.IsInterior() )
+            if ( face_type==FaceType::Interior && face.IsInterior() )
             {
                PermuteAndSetFaceDofsScatterIndices2(face,f_ind);
             }
-            else if ( type==FaceType::Boundary && face.IsBoundary() )
+            else if ( face_type==FaceType::Boundary && face.IsBoundary() )
             {
                SetBoundaryDofsScatterIndices2(face,f_ind);
             }
@@ -1305,7 +1415,9 @@ void L2FaceRestriction::ComputeScatterIndicesAndOffsets()
    }
 }
 
-void L2FaceRestriction::ComputeGatherIndices()
+void L2FaceRestriction::ComputeGatherIndices(
+   const ElementDofOrdering f_ordering,
+   const FaceType face_type)
 {
    Mesh &mesh = *fes.GetMesh();
    // Computation of gather_indices
@@ -1315,11 +1427,11 @@ void L2FaceRestriction::ComputeGatherIndices()
       Mesh::FaceInformation face = mesh.GetFaceInformation(f);
       MFEM_ASSERT(!face.IsShared(),
                   "Unexpected shared face in L2FaceRestriction.");
-      if ( face.IsOfFaceType(type) )
+      if ( face.IsOfFaceType(face_type) )
       {
          SetFaceDofsGatherIndices1(face,f_ind);
          if ( m==L2FaceValues::DoubleValued &&
-              type==FaceType::Interior &&
+              face_type==FaceType::Interior &&
               face.IsLocal())
          {
             PermuteAndSetFaceDofsGatherIndices2(face,f_ind);
@@ -1486,28 +1598,6 @@ void L2FaceRestriction::PermuteAndSetFaceDofsGatherIndices2(
    }
 }
 
-void L2FaceRestriction::NormalDerivativeMult(const Vector &x, Vector &y) const
-{
-   EnsureNormalDerivativeRestriction();
-   normal_deriv_restr->Mult(x, y);
-}
-
-void L2FaceRestriction::NormalDerivativeAddMultTranspose(const Vector &x,
-                                                         Vector &y) const
-{
-   EnsureNormalDerivativeRestriction();
-   normal_deriv_restr->AddMultTranspose(x, y);
-}
-
-void L2FaceRestriction::EnsureNormalDerivativeRestriction() const
-{
-   if (!normal_deriv_restr)
-   {
-      normal_deriv_restr.reset(
-         new L2NormalDerivativeFaceRestriction(fes, ordering, type));
-   }
-}
-
 InterpolationManager::InterpolationManager(const FiniteElementSpace &fes,
                                            ElementDofOrdering ordering,
                                            FaceType type)
@@ -1685,11 +1775,11 @@ NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
    if (!build) { return; }
    x_interp.UseDevice(true);
 
-   CheckFESpace();
+   CheckFESpace(f_ordering);
 
-   ComputeScatterIndicesAndOffsets();
+   ComputeScatterIndicesAndOffsets(f_ordering, type);
 
-   ComputeGatherIndices();
+   ComputeGatherIndices(f_ordering, type);
 }
 
 NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
@@ -2169,16 +2259,18 @@ int ToLexOrdering(const int dim, const int face_id, const int size1d,
       case 1:
          return 0;
       case 2:
-         return internal::ToLexOrdering2D(face_id, size1d, index);
+         return ToLexOrdering2D(face_id, size1d, index);
       case 3:
-         return internal::ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
+         return ToLexOrdering3D(face_id, size1d, index%size1d, index/size1d);
       default:
          MFEM_ABORT("Unsupported dimension.");
          return 0;
    }
 }
 
-void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
+void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
+   const ElementDofOrdering f_ordering,
+   const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
 
@@ -2242,7 +2334,9 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets()
    interpolations.InitializeNCInterpConfig();
 }
 
-void NCL2FaceRestriction::ComputeGatherIndices()
+void NCL2FaceRestriction::ComputeGatherIndices(
+   const ElementDofOrdering f_ordering,
+   const FaceType type)
 {
    Mesh &mesh = *fes.GetMesh();
    // Computation of gather_indices
@@ -2282,33 +2376,4 @@ void NCL2FaceRestriction::ComputeGatherIndices()
    gather_offsets[0] = 0;
 }
 
-Vector GetLVectorFaceNbrData(
-   const FiniteElementSpace &fes, const Vector &x, FaceType ftype)
-{
-#ifdef MFEM_USE_MPI
-   if (ftype == FaceType::Interior)
-   {
-      if (auto *pfes = const_cast<ParFiniteElementSpace*>
-                       (dynamic_cast<const ParFiniteElementSpace*>(&fes)))
-      {
-         if (auto *x_gf = const_cast<ParGridFunction*>
-                          (dynamic_cast<const ParGridFunction*>(&x)))
-         {
-            Vector &gf_face_nbr = x_gf->FaceNbrData();
-            if (gf_face_nbr.Size() == 0) { x_gf->ExchangeFaceNbrData(); }
-            gf_face_nbr.Read();
-            return Vector(gf_face_nbr, 0, gf_face_nbr.Size());
-         }
-         else
-         {
-            ParGridFunction gf(pfes, const_cast<Vector&>(x));
-            gf.ExchangeFaceNbrData();
-            return std::move(gf.FaceNbrData());
-         }
-      }
-   }
-#endif
-   return Vector();
-}
-
 } // namespace mfem

fem/restriction.hpp

@@ -14,7 +14,6 @@
 
 #include "../linalg/operator.hpp"
 #include "../mesh/mesh.hpp"
-#include "normal_deriv_restriction.hpp"
 
 namespace mfem
 {
@@ -22,8 +21,6 @@ namespace mfem
 class FiniteElementSpace;
 enum class ElementDofOrdering;
 
-class FaceQuadratureSpace;
-
 /// Abstract base class that defines an interface for element restrictions.
 class ElementRestrictionOperator : public Operator
 {
@@ -223,45 +220,6 @@ public:
       y = 0.0;
       AddMultTranspose(x, y);
    }
-
-   /** @brief For each face, sets @a y to the partial derivative of @a x with
-              respect to the reference coordinate whose direction is
-              perpendicular to the face on the reference element.
-
-    @details This is not the normal derivative in physical coordinates, but can
-             be mapped to the physical normal derivative using the element
-             Jacobian and the tangential derivatives (in reference coordinates)
-             which can be computed from the face values (provided by Mult).
-
-             Note that due to the polynomial degree of the element mapping, the
-             physical normal derivative may be a higher degree polynomial than
-             the restriction of the values to the face. However, the normal
-             derivative in reference coordinates has degree-1, and therefore can
-             be exactly represented with the degrees of freedom of a face
-             E-vector.
-
-    @param[in]     x The L-vector degrees of freedom.
-    @param[in,out] y The reference normal derivative degrees of freedom. Is
-                     E-vector like.
-    */
-   virtual void NormalDerivativeMult(const Vector &x, Vector &y) const
-   {
-      MFEM_ABORT("Not implemented for this restriction operator.");
-   }
-
-   /** @brief Add the face reference-normal derivative degrees of freedom in @a
-              x to the element degrees of freedom in @a y.
-
-       @details see NormalDerivativeMult.
-
-       @param[in]     x The degrees of freedom of the face reference-normal
-                        derivative. Is E-vector like.
-       @param[in,out] y The L-vector degrees of freedom.
-   */
-   virtual void NormalDerivativeAddMultTranspose(const Vector &x, Vector &y) const
-   {
-      MFEM_ABORT("Not implemented for this restriction operator.");
-   }
 };
 
 /// @brief Operator that extracts face degrees of freedom for H1, ND, or RT
@@ -407,7 +365,6 @@ class L2FaceRestriction : public FaceRestriction
 {
 protected:
    const FiniteElementSpace &fes;
-   const ElementDofOrdering ordering;
    const int nf; // Number of faces of the requested type
    const int ne; // Number of elements
    const int vdim; // vdim
@@ -422,7 +379,6 @@ protected:
    Array<int> scatter_indices2; // Scattering indices for element 2 on each face
    Array<int> gather_offsets; // offsets for the gathering indices of each dof
    Array<int> gather_indices; // gathering indices for each dof
-   mutable std::unique_ptr<L2NormalDerivativeFaceRestriction> normal_deriv_restr;
 
    /** @brief Constructs an L2FaceRestriction.
 
@@ -531,49 +487,34 @@ public:
    virtual void AddFaceMatricesToElementMatrices(const Vector &fea_data,
                                                  Vector &ea_data) const;
 
-   /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
-      face E-Vector.
-
-      @param[in]  x The L-vector degrees of freedom.
-      @param[out] y The face E-Vector degrees of freedom with the given format:
-                    (face_dofs x vdim x 2 x nf) where nf is the number of
-                    interior or boundary faces requested by @a type in the
-                    constructor. The face_dofs are ordered according to the
-                    given ElementDofOrdering. */
-   void NormalDerivativeMult(const Vector &x, Vector &y) const override;
-
-   /** @brief Add the face reference-normal derivative degrees of freedom in @a
-              x to the element degrees of freedom in @a y.
-
-       @details see NormalDerivativeMult.
-
-       @param[in]     x The degrees of freedom of the face reference-normal
-                        derivative. Is E-vector like.
-       @param[in,out] y The L-vector degrees of freedom.
-   */
-   void NormalDerivativeAddMultTranspose(const Vector &x,
-                                         Vector &y) const override;
 private:
    /** @brief Compute the scatter indices: L-vector to E-vector, and the offsets
        for the gathering: E-vector to L-vector.
+
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets();
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
+                                        const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
-   */
-   void ComputeGatherIndices();
 
-   /// Create the internal normal derivative restriction operator if needed.
-   void EnsureNormalDerivativeRestriction() const;
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
+   */
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
+                             const FaceType type);
 
 protected:
    mutable Array<int> face_map; // Used in the computation of GetFaceDofs
 
    /** @brief Verify that L2FaceRestriction is built from an L2 FESpace.
+
+       @param[in] f_ordering The requested face dof ordering.
    */
-   void CheckFESpace();
+   void CheckFESpace(const ElementDofOrdering f_ordering);
 
    /** @brief Set the scattering indices of elem1, and increment the offsets for
        the face described by the @a face. The ordering of the face dofs of elem1
@@ -997,14 +938,22 @@ private:
    /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
        for the gathering: E-vector to L-vector, and the interpolators from
        coarse to fine face for master non-comforming faces.
+
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeScatterIndicesAndOffsets();
+   void ComputeScatterIndicesAndOffsets(const ElementDofOrdering f_ordering,
+                                        const FaceType type);
 
    /** @brief Compute the gather indices: E-vector to L-vector.
 
        Note: Requires the gather offsets to be computed.
+
+       @param[in] f_ordering Request a specific face dof ordering.
+       @param[in] type       Request internal or boundary faces dofs.
    */
-   void ComputeGatherIndices();
+   void ComputeGatherIndices(const ElementDofOrdering f_ordering,
+                             const FaceType type);
 
 public:
    /** @brief Scatter the degrees of freedom, i.e. goes from L-Vector to
@@ -1066,6 +1015,7 @@ public:
    void DoubleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
 };
 
+
 /** @brief Convert a dof face index from Native ordering to lexicographic
     ordering for quads and hexes.
 
@@ -1094,20 +1044,6 @@ int PermuteFaceL2(const int dim, const int face_id1,
                   const int face_id2, const int orientation,
                   const int size1d, const int index);
 
-/// @brief Return the face-neighbor data given the L-vector @a x.
-///
-/// If the input vector @a x is a ParGridFunction with non-empty face-neighbor
-/// data, return an alias to ParGridFunction::FaceNbrData() (avoiding an
-/// unneeded call to ParGridFunction::ExchangeFaceNbrData).
-///
-/// Otherwise, create a temporary ParGridFunction, exchange the face-neighbor
-/// data, and return the resulting vector.
-///
-/// If @a fes is not a parallel space, or if @a ftype is not FaceType::Interior,
-/// return an empty vector.
-Vector GetLVectorFaceNbrData(
-   const FiniteElementSpace &fes, const Vector &x, FaceType ftype);
-
 }
 
 #endif // MFEM_RESTRICTION

fem/tmop.cpp

@@ -2949,15 +2949,6 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &s0,
    MFEM_VERIFY(surf_fit_pos == NULL,
                "Using both fitting approaches is not supported.");
 
-   const int dim = s0.FESpace()->GetMesh()->Dimension();
-   Mesh *mesh = s0.FESpace()->GetMesh();
-   MFEM_VERIFY(mesh->GetNodes()->Size() == dim*s0.Size(),
-               "Mesh and level-set polynomial order must be the same.");
-   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
-                                (s0.FESpace()->FEColl());
-   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
-               "grid function.");
-
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
    surf_fit_gf->CountElementsPerVDof(surf_fit_dof_count);
@@ -2996,24 +2987,12 @@ void TMOP_Integrator::EnableSurfaceFitting(const GridFunction &pos,
 void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
                                            const Array<bool> &smarker,
                                            Coefficient &coeff,
-                                           AdaptivityEvaluator &ae,
-                                           AdaptivityEvaluator *aegrad,
-                                           AdaptivityEvaluator *aehess)
+                                           AdaptivityEvaluator &ae)
 {
    // To have both we must duplicate the markers.
    MFEM_VERIFY(surf_fit_pos == NULL,
                "Using both fitting approaches is not supported.");
 
-   const int dim = s0.FESpace()->GetMesh()->Dimension();
-   ParMesh *pmesh = s0.ParFESpace()->GetParMesh();
-   MFEM_VERIFY(pmesh->GetNodes()->Size() == dim*s0.Size(),
-               "Mesh and level-set polynomial order must be the same.");
-   const H1_FECollection *fec = dynamic_cast<const H1_FECollection *>
-                                (s0.FESpace()->FEColl());
-   MFEM_VERIFY(fec, "Only H1_FECollection is supported for the surface fitting "
-               "grid function.");
-
-
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
    s0.CountElementsPerVDof(surf_fit_dof_count);
@@ -3021,80 +3000,11 @@ void TMOP_Integrator::EnableSurfaceFitting(const ParGridFunction &s0,
    surf_fit_coeff = &coeff;
    surf_fit_eval = &ae;
 
-   surf_fit_eval->SetParMetaInfo(*pmesh, *s0.ParFESpace());
+   surf_fit_eval->SetParMetaInfo(*s0.ParFESpace()->GetParMesh(),
+                                 *s0.ParFESpace());
    surf_fit_eval->SetInitialField
    (*surf_fit_gf->FESpace()->GetMesh()->GetNodes(), *surf_fit_gf);
-
-   if (!aegrad) { return; }
-
-   MFEM_VERIFY(aehess, "AdaptivityEvaluator for Hessians must be provided too.");
-
-   ParFiniteElementSpace *fes = s0.ParFESpace();
-
-   // FE space for gradients.
-   delete surf_fit_grad;
-   H1_FECollection *fec_grad = new H1_FECollection(fec->GetOrder(), dim,
-                                                   fec->GetBasisType());
-   ParFiniteElementSpace *fes_grad = new ParFiniteElementSpace(pmesh, fec_grad,
-                                                               dim);
-   // Initial gradients.
-   surf_fit_grad = new GridFunction(fes_grad);
-   surf_fit_grad->MakeOwner(fec_grad);
-   for (int d = 0; d < dim; d++)
-   {
-      ParGridFunction surf_fit_grad_comp(fes, surf_fit_grad->GetData()+d*s0.Size());
-      s0.GetDerivative(1, d, surf_fit_grad_comp);
-   }
-   surf_fit_eval_grad = aegrad;
-   surf_fit_eval_grad->SetParMetaInfo(*pmesh, *fes_grad);
-   surf_fit_eval_grad->SetInitialField(*pmesh->GetNodes(), *surf_fit_grad);
-
-   // FE space for Hessians.
-   delete surf_fit_hess;
-   H1_FECollection *fec_hess = new H1_FECollection(fec->GetOrder(), dim,
-                                                   fec->GetBasisType());
-   ParFiniteElementSpace *fes_hess = new ParFiniteElementSpace(pmesh, fec_hess,
-                                                               dim*dim);
-   // Initial Hessians.
-   surf_fit_hess = new GridFunction(fes_hess);
-   surf_fit_hess->MakeOwner(fec_hess);
-   int id = 0;
-   for (int d = 0; d < dim; d++)
-   {
-      for (int idir = 0; idir < dim; idir++)
-      {
-         ParGridFunction surf_fit_grad_comp(fes,
-                                            surf_fit_grad->GetData()+d*s0.Size());
-         ParGridFunction surf_fit_hess_comp(fes,
-                                            surf_fit_hess->GetData()+id*s0.Size());
-         surf_fit_grad_comp.GetDerivative(1, idir, surf_fit_hess_comp);
-         id++;
-      }
-   }
-   surf_fit_eval_hess = aehess;
-   surf_fit_eval_hess->SetParMetaInfo(*pmesh, *fes_hess);
-   surf_fit_eval_hess->SetInitialField(*pmesh->GetNodes(), *surf_fit_hess);
-
-   // Store DOF indices that are marked for fitting. Used to reduce work for
-   // transferring information between source/background and current mesh.
-   surf_fit_marker_dof_index.SetSize(0);
-#ifdef MFEM_USE_GSLIB
-   if (dynamic_cast<InterpolatorFP *>(surf_fit_eval) &&
-       dynamic_cast<InterpolatorFP *>(surf_fit_eval_grad) &&
-       dynamic_cast<InterpolatorFP *>(surf_fit_eval_hess))
-   {
-      for (int i = 0; i < surf_fit_marker->Size(); i++)
-      {
-         if ((*surf_fit_marker)[i] == true)
-         {
-            surf_fit_marker_dof_index.Append(i);
-         }
-      }
-   }
-#endif
-
-   *surf_fit_grad = 0.0;
-   *surf_fit_hess = 0.0;
+   surf_fit_gf_bg = false;
 }
 
 void TMOP_Integrator::EnableSurfaceFittingFromSource(
@@ -3112,17 +3022,16 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
    // Setup for level set function
    delete surf_fit_gf;
    surf_fit_gf = new GridFunction(s0);
+   *surf_fit_gf = 0.0;
    surf_fit_marker = &smarker;
    surf_fit_coeff = &coeff;
    surf_fit_eval = &ae;
 
+   surf_fit_gf_bg = true;
    surf_fit_eval->SetParMetaInfo(*s_bg.ParFESpace()->GetParMesh(),
                                  *s_bg.ParFESpace());
    surf_fit_eval->SetInitialField
    (*s_bg.FESpace()->GetMesh()->GetNodes(), s_bg);
-   GridFunction *nodes = s0.FESpace()->GetMesh()->GetNodes();
-   surf_fit_eval->ComputeAtNewPosition(*nodes, *surf_fit_gf,
-                                       nodes->FESpace()->GetOrdering());
 
    // Setup for gradient on background mesh
    MFEM_VERIFY(s_bg_grad.ParFESpace()->GetOrdering() ==
@@ -3132,11 +3041,11 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
    delete surf_fit_grad;
    surf_fit_grad = new GridFunction(s0_grad);
    *surf_fit_grad = 0.0;
-   surf_fit_eval_grad = &age;
-   surf_fit_eval_hess = &ahe;
-   surf_fit_eval_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
-                                      *s_bg_grad.ParFESpace());
-   surf_fit_eval_grad->SetInitialField
+   surf_fit_eval_bg_grad = &age;
+   surf_fit_eval_bg_hess = &ahe;
+   surf_fit_eval_bg_grad->SetParMetaInfo(*s_bg_grad.ParFESpace()->GetParMesh(),
+                                         *s_bg_grad.ParFESpace());
+   surf_fit_eval_bg_grad->SetInitialField
    (*s_bg_grad.FESpace()->GetMesh()->GetNodes(), s_bg_grad);
 
    // Setup for Hessian on background mesh
@@ -3147,9 +3056,9 @@ void TMOP_Integrator::EnableSurfaceFittingFromSource(
    delete surf_fit_hess;
    surf_fit_hess = new GridFunction(s0_hess);
    *surf_fit_hess = 0.0;
-   surf_fit_eval_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
-                                      *s_bg_hess.ParFESpace());
-   surf_fit_eval_hess->SetInitialField
+   surf_fit_eval_bg_hess->SetParMetaInfo(*s_bg_hess.ParFESpace()->GetParMesh(),
+                                         *s_bg_hess.ParFESpace());
+   surf_fit_eval_bg_hess->SetInitialField
    (*s_bg_hess.FESpace()->GetMesh()->GetNodes(), s_bg_hess);
 
    // Count number of zones that share each of the DOFs
@@ -3954,7 +3863,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
 
    Vector sigma_e(dof_s);
    DenseMatrix surf_fit_grad_e(dof_s, dim);
-   if (surf_fit_gf)
+   if (surf_fit_gf || surf_fit_gf_bg)
    {
       surf_fit_gf->GetSubVector(vdofs, sigma_e);
 
@@ -3962,7 +3871,7 @@ void TMOP_Integrator::AssembleElemVecSurfFit(const FiniteElement &el_x,
       // The FE coefficients of the gradient go in surf_fit_grad_e.
       Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
       DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_grad)
+      if (surf_fit_gf_bg)
       {
          surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
          surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -4036,7 +3945,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
    Vector sigma_e(dof_s);
    DenseMatrix surf_fit_grad_e(dof_s, dim);
    DenseMatrix surf_fit_hess_e(dof_s, dim*dim);
-   if (surf_fit_gf)
+   if (surf_fit_gf || surf_fit_gf_bg)
    {
       surf_fit_gf->GetSubVector(vdofs, sigma_e);
 
@@ -4044,7 +3953,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
       // The FE coefficients of the gradient go in surf_fit_grad_e.
       Vector grad_ptr(surf_fit_grad_e.GetData(), dof_s * dim);
       DenseMatrix grad_phys; // This will be (dof x dim, dof).
-      if (surf_fit_grad)
+      if (surf_fit_gf_bg)
       {
          surf_fit_grad->FESpace()->GetElementVDofs(el_id, dofs);
          surf_fit_grad->GetSubVector(dofs, grad_ptr);
@@ -4058,7 +3967,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
       // Project the Hessian of sigma in the same space.
       // The FE coefficients of the Hessian go in surf_fit_hess_e.
       Vector hess_ptr(surf_fit_hess_e.GetData(), dof_s*dim*dim);
-      if (surf_fit_hess)
+      if (surf_fit_gf_bg)
       {
          surf_fit_hess->FESpace()->GetElementVDofs(el_id, dofs);
          surf_fit_hess->GetSubVector(dofs, hess_ptr);
@@ -4085,7 +3994,7 @@ void TMOP_Integrator::AssembleElemGradSurfFit(const FiniteElement &el_x,
       Tpr.SetIntPoint(&ip);
       real_t w = surf_fit_normal * surf_fit_coeff->Eval(Tpr, ip);
 
-      if (surf_fit_gf)
+      if (surf_fit_gf || surf_fit_gf_bg)
       {
          Vector gg_ptr(surf_fit_hess_s.GetData(), dim * dim);
          surf_fit_hess_e.GetRow(s, gg_ptr);
@@ -4467,130 +4376,6 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
    dx = detv_avg_min / dxscale;
 }
 
-void TMOP_Integrator::RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
-                                                         int new_x_ordering)
-{
-   if (!surf_fit_gf) { return; }
-
-   if (surf_fit_marker_dof_index.Size())
-   {
-      // Interpolate information only at DOFs marked for fitting.
-      const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
-      const int cnt = surf_fit_marker_dof_index.Size();
-      const int total_cnt = new_x.Size()/dim;
-      Vector new_x_sorted(cnt*dim);
-      if (new_x_ordering == 0)
-      {
-         for (int d = 0; d < dim; d++)
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               new_x_sorted(i + d*cnt) = new_x(dof_index + d*total_cnt);
-            }
-         }
-      }
-      else
-      {
-         for (int i = 0; i < cnt; i++)
-         {
-            int dof_index = surf_fit_marker_dof_index[i];
-            for (int d = 0; d < dim; d++)
-            {
-               new_x_sorted(d + i*dim) = new_x(d + dof_index*dim);
-            }
-         }
-      }
-
-      // Interpolate values of the LS.
-      Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
-      surf_fit_eval->ComputeAtNewPosition(new_x_sorted, surf_fit_gf_int,
-                                          new_x_ordering);
-      for (int i = 0; i < cnt; i++)
-      {
-         int dof_index = surf_fit_marker_dof_index[i];
-         (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
-      }
-
-      // Interpolate gradients of the LS.
-      surf_fit_eval_grad->ComputeAtNewPosition(new_x_sorted, surf_fit_grad_int,
-                                               new_x_ordering);
-      // Assumes surf_fit_grad and surf_fit_gf share the same space
-      const int grad_dim = surf_fit_grad->VectorDim();
-      const int grad_cnt = surf_fit_grad->Size()/grad_dim;
-      if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
-      {
-         for (int d = 0; d < grad_dim; d++)
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               (*surf_fit_grad)[dof_index + d*grad_cnt] =
-                  surf_fit_grad_int(i + d*cnt);
-            }
-         }
-      }
-      else
-      {
-         for (int i = 0; i < cnt; i++)
-         {
-            int dof_index = surf_fit_marker_dof_index[i];
-            for (int d = 0; d < grad_dim; d++)
-            {
-               (*surf_fit_grad)[dof_index*grad_dim + d] =
-                  surf_fit_grad_int(i*grad_dim + d);
-            }
-         }
-      }
-
-      // Interpolate Hessians of the LS.
-      surf_fit_eval_hess->ComputeAtNewPosition(new_x_sorted, surf_fit_hess_int,
-                                               new_x_ordering);
-      // Assumes surf_fit_hess and surf_fit_gf share the same space
-      const int hess_dim = surf_fit_hess->VectorDim();
-      const int hess_cnt = surf_fit_hess->Size()/hess_dim;
-      if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
-      {
-         for (int d = 0; d < hess_dim; d++)
-         {
-            for (int i = 0; i < cnt; i++)
-            {
-               int dof_index = surf_fit_marker_dof_index[i];
-               (*surf_fit_hess)[dof_index + d*hess_cnt] =
-                  surf_fit_hess_int(i + d*cnt);
-            }
-         }
-      }
-      else
-      {
-         for (int i = 0; i < cnt; i++)
-         {
-            int dof_index = surf_fit_marker_dof_index[i];
-            for (int d = 0; d < hess_dim; d++)
-            {
-               (*surf_fit_hess)[dof_index*hess_dim + d] =
-                  surf_fit_hess_int(i*hess_dim + d);
-            }
-         }
-      }
-
-   }
-   else
-   {
-      surf_fit_eval->ComputeAtNewPosition(new_x, *surf_fit_gf, new_x_ordering);
-      if (surf_fit_eval_grad)
-      {
-         surf_fit_eval_grad->ComputeAtNewPosition(new_x, *surf_fit_grad,
-                                                  new_x_ordering);
-      }
-      if (surf_fit_eval_hess)
-      {
-         surf_fit_eval_hess->ComputeAtNewPosition(new_x, *surf_fit_hess,
-                                                  new_x_ordering);
-      }
-   }
-}
-
 void TMOP_Integrator::
 UpdateAfterMeshPositionChange(const Vector &x_new,
                               const FiniteElementSpace &x_fes)
@@ -4621,11 +4406,112 @@ UpdateAfterMeshPositionChange(const Vector &x_new,
       adapt_lim_eval->ComputeAtNewPosition(x_new, *adapt_lim_gf, ordering);
    }
 
-   // Update surf_fit_gf (and optionally its gradients) if surface
-   // fitting is enabled.
+   // Update surf_fit_gf if surface fitting is enabled.
    if (surf_fit_gf)
    {
-      RemapSurfaceFittingLevelSetAtNodes(x_new, ordering);
+      if (surf_fit_gf_bg)
+      {
+         // Interpolate information for only DOFs marked for fitting.
+         const int dim = surf_fit_gf->FESpace()->GetMesh()->Dimension();
+         const int cnt = surf_fit_marker_dof_index.Size();
+         const int total_cnt = x_new.Size()/dim;
+         Vector new_x_sorted(cnt*dim);
+         if (ordering == 0)
+         {
+            for (int d = 0; d < dim; d++)
+            {
+               for (int i = 0; i < cnt; i++)
+               {
+                  int dof_index = surf_fit_marker_dof_index[i];
+                  new_x_sorted(i + d*cnt) = x_new(dof_index + d*total_cnt);
+               }
+            }
+         }
+         else
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               for (int d = 0; d < dim; d++)
+               {
+                  new_x_sorted(d + i*dim) = x_new(d + dof_index*dim);
+               }
+            }
+         }
+
+         Vector surf_fit_gf_int, surf_fit_grad_int, surf_fit_hess_int;
+         surf_fit_eval->ComputeAtNewPosition(
+            new_x_sorted, surf_fit_gf_int, ordering);
+         for (int i = 0; i < cnt; i++)
+         {
+            int dof_index = surf_fit_marker_dof_index[i];
+            (*surf_fit_gf)[dof_index] = surf_fit_gf_int(i);
+         }
+
+         surf_fit_eval_bg_grad->ComputeAtNewPosition(
+            new_x_sorted, surf_fit_grad_int, ordering);
+         // Assumes surf_fit_grad and surf_fit_gf share the same space
+         const int grad_dim = surf_fit_grad->VectorDim();
+         const int grad_cnt = surf_fit_grad->Size()/grad_dim;
+         if (surf_fit_grad->FESpace()->GetOrdering() == Ordering::byNODES)
+         {
+            for (int d = 0; d < grad_dim; d++)
+            {
+               for (int i = 0; i < cnt; i++)
+               {
+                  int dof_index = surf_fit_marker_dof_index[i];
+                  (*surf_fit_grad)[dof_index + d*grad_cnt] =
+                     surf_fit_grad_int(i + d*cnt);
+               }
+            }
+         }
+         else
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               for (int d = 0; d < grad_dim; d++)
+               {
+                  (*surf_fit_grad)[dof_index*dim + d] =
+                     surf_fit_grad_int(i*dim + d);
+               }
+            }
+         }
+
+         surf_fit_eval_bg_hess->ComputeAtNewPosition(
+            new_x_sorted, surf_fit_hess_int, ordering);
+         // Assumes surf_fit_hess and surf_fit_gf share the same space
+         const int hess_dim = surf_fit_hess->VectorDim();
+         const int hess_cnt = surf_fit_hess->Size()/hess_dim;
+         if (surf_fit_hess->FESpace()->GetOrdering() == Ordering::byNODES)
+         {
+            for (int d = 0; d < hess_dim; d++)
+            {
+               for (int i = 0; i < cnt; i++)
+               {
+                  int dof_index = surf_fit_marker_dof_index[i];
+                  (*surf_fit_hess)[dof_index + d*hess_cnt] =
+                     surf_fit_hess_int(i + d*cnt);
+               }
+            }
+         }
+         else
+         {
+            for (int i = 0; i < cnt; i++)
+            {
+               int dof_index = surf_fit_marker_dof_index[i];
+               for (int d = 0; d < hess_dim; d++)
+               {
+                  (*surf_fit_hess)[dof_index*dim + d] =
+                     surf_fit_hess_int(i*dim + d);
+               }
+            }
+         }
+      }
+      else
+      {
+         surf_fit_eval->ComputeAtNewPosition(x_new, *surf_fit_gf, ordering);
+      }
    }
 }
 

fem/tmop.hpp

@@ -1784,11 +1784,12 @@ protected:
    // Fitting to given physical positions.
    TMOP_QuadraticLimiter *surf_fit_limiter; // Owned. Created internally.
    const GridFunction *surf_fit_pos;        // Not owned. Positions to fit.
-   real_t surf_fit_normal;                  // Normalization factor.
-   GridFunction *surf_fit_grad, *surf_fit_hess; // Owned. Created internally.
-   AdaptivityEvaluator *surf_fit_eval_grad, *surf_fit_eval_hess; // Not owned.
-   Array<int> surf_fit_dof_count;            // Number of dofs per node.
-   Array<int> surf_fit_marker_dof_index;     // Indices of nodes to fit.
+   real_t surf_fit_normal;
+   bool surf_fit_gf_bg;
+   GridFunction *surf_fit_grad, *surf_fit_hess;
+   AdaptivityEvaluator *surf_fit_eval_bg_grad, *surf_fit_eval_bg_hess;
+   Array<int> surf_fit_dof_count;
+   Array<int> surf_fit_marker_dof_index;
 
    DiscreteAdaptTC *discr_tc;
 
@@ -1984,10 +1985,6 @@ protected:
    real_t ComputeUntanglerMaxMuBarrier(const Vector &x,
                                        const FiniteElementSpace &fes);
 
-   // Remaps the internal surface fitting gridfunction object at provided
-   // locations.
-   void RemapSurfaceFittingLevelSetAtNodes(const Vector &new_x,
-                                           int new_x_ordering);
 public:
    /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
        @param[in] tc   Target-matrix construction algorithm to use (not owned).
@@ -2003,8 +2000,9 @@ public:
         surf_fit_marker(NULL), surf_fit_coeff(NULL),
         surf_fit_gf(NULL), surf_fit_eval(NULL),
         surf_fit_limiter(NULL), surf_fit_pos(NULL),
-        surf_fit_normal(1.0), surf_fit_grad(NULL), surf_fit_hess(NULL),
-        surf_fit_eval_grad(NULL), surf_fit_eval_hess(NULL),
+        surf_fit_normal(1.0),
+        surf_fit_gf_bg(false), surf_fit_grad(NULL), surf_fit_hess(NULL),
+        surf_fit_eval_bg_grad(NULL), surf_fit_eval_bg_hess(NULL),
         discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
    { PA.enabled = false; }
@@ -2105,15 +2103,9 @@ public:
 
 #ifdef MFEM_USE_MPI
    /// Parallel support for surface fitting to the zero level set of a function.
-   /// Here, we add two optional inputs: @a aegrad and @a aehess. When provided,
-   /// the first and second derivative of the input level set are computed on
-   /// the initial mesh, and @a aegrad and @a aehess are used to remap grad_s(x)
-   /// from grad_s0(x0) and hess_s(x) from hess_s0(x0), respectively.
    void EnableSurfaceFitting(const ParGridFunction &s0,
                              const Array<bool> &smarker, Coefficient &coeff,
-                             AdaptivityEvaluator &ae,
-                             AdaptivityEvaluator *aegrad = NULL,
-                             AdaptivityEvaluator *aehess = NULL);
+                             AdaptivityEvaluator &ae);
 
    /** @brief Fitting of certain DOFs in the current mesh to the zero level set
        of a function defined on another (finer) source mesh.

fem/tmop_tools.cpp

@@ -429,13 +429,11 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 #endif
 
    real_t scale = 1.0;
-   bool fitting = IsSurfaceFittingEnabled();
-   real_t init_fit_avg_err, init_fit_max_err = 0.0;
-   if (fitting && surf_fit_converge_error)
+   real_t avg_surf_fit_err, max_surf_fit_err = 0.0;
+   if (surf_fit_max_threshold > 0.0)
    {
-      GetSurfaceFittingError(x_out_loc, init_fit_avg_err, init_fit_max_err);
-      // Check for convergence
-      if (init_fit_max_err < surf_fit_max_err_limit)
+      GetSurfaceFittingError(x_out_loc, avg_surf_fit_err, max_surf_fit_err);
+      if (max_surf_fit_err < surf_fit_max_threshold)
       {
          if (print_options.iterations)
          {
@@ -446,12 +444,11 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
          return scale;
       }
    }
-
-   if (surf_fit_adapt_count >= surf_fit_adapt_count_limit)
+   if (adapt_inc_count >= max_adapt_inc_count)
    {
       if (print_options.iterations)
       {
-         mfem::out << "TMOPNewtonSolver terminated "
+         mfem::out << "TMOPNewtonSolver converged "
                    "based on max number of times surface fitting weight can"
                    "be increased. \n";
       }
@@ -470,7 +467,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       // reference to detect deteriorations.
       MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
                   " intermediate mesh is invalid. Contact TMOP Developers.");
-      MFEM_VERIFY(min_detJ_limit == 0.0,
+      MFEM_VERIFY(min_detJ_threshold == 0.0,
                   "This setup is not supported. Contact TMOP Developers.");
       *min_det_ptr = untangle_factor * min_detT_in;
    }
@@ -481,7 +478,6 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
    bool x_out_ok = false;
    real_t energy_out = 0.0, min_detT_out;
    const real_t norm_in = Norm(r);
-   real_t avg_fit_err, max_fit_err = 0.0;
 
    const real_t detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
    compute_metric_quantile_flag = false;
@@ -492,9 +488,6 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
    // Perform the line search.
    for (int i = 0; i < 12; i++)
    {
-      avg_fit_err = 0.0;
-      max_fit_err = 0.0;
-
       // Update the mesh and get the L-vector in x_out_loc.
       add(x, -scale, c, x_out);
       if (serial)
@@ -509,7 +502,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
       // Check the changes in detJ.
       min_detT_out = ComputeMinDet(x_out_loc, *fes);
-      if (untangling == false && min_detT_out <= min_detJ_limit)
+      if (untangling == false && min_detT_out <= min_detJ_threshold)
       {
          // No untangling, and detJ got negative (or small) -- no good.
          if (print_options.iterations)
@@ -536,19 +529,18 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       // Check the changes in total energy.
       ProcessNewState(x_out);
 
-      // Ensure sufficient decrease in fitting error if we are trying to
-      // converge based on error.
-      if (fitting && surf_fit_converge_error)
+      real_t avg_fit_err, max_fit_err = 0.0;
+      if (surf_fit_max_threshold > 0.0)
       {
          GetSurfaceFittingError(x_out_loc, avg_fit_err, max_fit_err);
-         if (max_fit_err >= 1.2*init_fit_max_err)
+      }
+      if (surf_fit_max_threshold > 0.0 && max_fit_err >= 1.2*max_surf_fit_err)
+      {
+         if (print_options.iterations)
          {
-            if (print_options.iterations)
-            {
-               mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
-            }
-            scale *= 0.5; continue;
+            mfem::out << "Scale = " << scale << " Surf fit err increased.\n";
          }
+         scale *= 0.5; continue;
       }
 
       if (serial)
@@ -622,7 +614,7 @@ real_t TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
 
    if (x_out_ok == false) { scale = 0.0; }
 
-   if (surf_fit_scale_factor > 0.0) { surf_fit_coeff_update = true; }
+   if (surf_fit_scale_factor > 0.0) { update_surf_fit_coeff = true; }
    compute_metric_quantile_flag = true;
 
    return scale;
@@ -665,7 +657,7 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
    for (int i = 0; i < integs.Size(); i++)
    {
       ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti && ti->IsSurfaceFittingEnabled())
+      if (ti)
       {
          weight = ti->GetSurfaceFittingWeight();
          weights.Append(weight);
@@ -676,11 +668,8 @@ void TMOPNewtonSolver::GetSurfaceFittingWeight(Array<real_t> &weights) const
          Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
          for (int j = 0; j < ati.Size(); j++)
          {
-            if (ati[j]->IsSurfaceFittingEnabled())
-            {
-               weight = ati[j]->GetSurfaceFittingWeight();
-               weights.Append(weight);
-            }
+            weight = ati[j]->GetSurfaceFittingWeight();
+            weights.Append(weight);
          }
       }
    }
@@ -727,39 +716,6 @@ void TMOPNewtonSolver::GetSurfaceFittingError(const Vector &x_loc,
    }
 }
 
-bool TMOPNewtonSolver::IsSurfaceFittingEnabled() const
-{
-   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
-   const Array<NonlinearFormIntegrator*> &integs = *nlf->GetDNFI();
-   TMOP_Integrator *ti  = NULL;
-   TMOPComboIntegrator *co = NULL;
-
-   for (int i = 0; i < integs.Size(); i++)
-   {
-      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
-      if (ti)
-      {
-         if (ti->IsSurfaceFittingEnabled())
-         {
-            return true;
-         }
-      }
-      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
-      if (co)
-      {
-         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
-         for (int j = 0; j < ati.Size(); j++)
-         {
-            if (ati[j]->IsSurfaceFittingEnabled())
-            {
-               return true;
-            }
-         }
-      }
-   }
-   return false;
-}
-
 void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
 {
    const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
@@ -845,46 +801,38 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
    // adaptive surface fitting is enabled. The idea is to increase the
    // coefficient if the surface fitting error does not sufficiently
    // decrease between subsequent TMOPNewtonSolver iterations.
-   if (surf_fit_coeff_update)
+   if (update_surf_fit_coeff)
    {
       // Get surface fitting errors.
-      GetSurfaceFittingError(x_loc, surf_fit_avg_err, surf_fit_max_err);
+      GetSurfaceFittingError(x_loc, surf_fit_err_avg, surf_fit_err_max);
       // Get array with surface fitting weights.
-      Array<real_t> fitweights;
-      GetSurfaceFittingWeight(fitweights);
+      Array<real_t> weights;
+      GetSurfaceFittingWeight(weights);
 
       if (print_options.iterations)
       {
          mfem::out << "Avg/Max surface fitting error: " <<
-                   surf_fit_avg_err << " " <<
-                   surf_fit_max_err << "\n";
+                   surf_fit_err_avg << " " <<
+                   surf_fit_err_max << "\n";
          mfem::out << "Min/Max surface fitting weight: " <<
-                   fitweights.Min() << " " << fitweights.Max() << "\n";
+                   weights.Min() << " " << weights.Max() << "\n";
       }
 
-      real_t change_surf_fit_err = surf_fit_avg_err_prvs-surf_fit_avg_err;
-      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_avg_err_prvs;
-
+      real_t change_surf_fit_err = surf_fit_err_avg_prvs-surf_fit_err_avg;
+      real_t rel_change_surf_fit_err = change_surf_fit_err/surf_fit_err_avg_prvs;
       // Increase the surface fitting coefficient if the surface fitting error
-      // does not decrease sufficiently. If we are converging based on residual,
-      // also make sure we have not reached the maximum fitting weight and
-      // error threshold.
-      if (rel_change_surf_fit_err < surf_fit_err_rel_change_limit &&
-          (surf_fit_converge_error ||
-           (fitweights.Max() < surf_fit_weight_limit &&
-            surf_fit_max_err > surf_fit_max_err_limit)))
+      // does not decrease sufficiently.
+      if (rel_change_surf_fit_err < surf_fit_rel_change_threshold)
       {
-         real_t scale_factor = std::min(surf_fit_scale_factor,
-                                        surf_fit_weight_limit/fitweights.Max());
-         UpdateSurfaceFittingWeight(scale_factor);
-         surf_fit_adapt_count += 1;
+         UpdateSurfaceFittingWeight(surf_fit_scale_factor);
+         adapt_inc_count += 1;
       }
       else
       {
-         surf_fit_adapt_count = 0;
+         adapt_inc_count = 0;
       }
-      surf_fit_avg_err_prvs = surf_fit_avg_err;
-      surf_fit_coeff_update = false;
+      surf_fit_err_avg_prvs = surf_fit_err_avg;
+      update_surf_fit_coeff = false;
    }
 }
 

fem/tmop_tools.hpp

@@ -134,20 +134,18 @@ protected:
    int solver_type;
    bool parallel;
 
-   // Line search step is rejected if min(detJ) <= min_detJ_limit.
-   real_t min_detJ_limit = 0.0;
+   // Line search step is rejected if min(detJ) <= min_detJ_threshold.
+   real_t min_detJ_threshold = 0.0;
 
    // Surface fitting variables.
-   mutable real_t surf_fit_avg_err_prvs = 10000.0;
-   mutable real_t surf_fit_avg_err, surf_fit_max_err;
-   mutable bool surf_fit_coeff_update = false;
-   real_t surf_fit_max_err_limit = -1.0;
-   real_t surf_fit_err_rel_change_limit = 0.001;
+   mutable real_t surf_fit_err_avg_prvs = 10000.0;
+   mutable real_t surf_fit_err_avg, surf_fit_err_max;
+   mutable bool update_surf_fit_coeff = false;
+   real_t surf_fit_max_threshold = -1.0;
+   real_t surf_fit_rel_change_threshold = 0.001;
    real_t surf_fit_scale_factor = 0.0;
-   mutable int surf_fit_adapt_count = 0;
-   mutable int surf_fit_adapt_count_limit = 10;
-   mutable real_t surf_fit_weight_limit = 1e10;
-   bool surf_fit_converge_error = false;
+   mutable int adapt_inc_count = 0;
+   mutable int max_adapt_inc_count = 10;
 
    // Minimum determinant over the whole mesh. Used for mesh untangling.
    real_t *min_det_ptr = nullptr;
@@ -193,9 +191,6 @@ protected:
    void GetSurfaceFittingWeight(Array<real_t> &weights) const;
    ///@}
 
-   /// Check if surface fitting is enabled.
-   bool IsSurfaceFittingEnabled() const;
-
 public:
 #ifdef MFEM_USE_MPI
    TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
@@ -229,94 +224,38 @@ public:
    /// (ii) surface fitting weight.
    virtual void ProcessNewState(const Vector &x) const;
 
-   /** @name Methods for adaptive surface fitting.
-       \brief These methods control the behavior of the weight and the
-       termination of the solver. (Experimental)
-
-       Adaptive fitting weight: The weight is modified after each
-       TMOPNewtonSolver iteration as:
-       w_{k+1} = w_{k} * \ref surf_fit_scale_factor if the relative
-       change in average fitting error < \ref surf_fit_err_rel_change_limit.
-       When converging based on the residual, we enforce the fitting weight
-       to be at-most \ref surf_fit_weight_limit, and increase it only if the
-       fitting error is below user prescribed threshold
-       (\ref surf_fit_max_err_limit).
-       See \ref SetAdaptiveSurfaceFittingScalingFactor and
-       \ref SetAdaptiveSurfaceFittingRelativeChangeThreshold.
-
-       Note that the solver stops if the maximum surface fitting error
-       does not sufficiently decrease for \ref surf_fit_adapt_count_limit (default 10)
-       consecutive increments of the fitting weight during weight adaptation.
-       This typically occurs when the mesh cannot align with the level-set
-       without degrading element quality.
-       See \ref SetMaxNumberofIncrementsForAdaptiveFitting.
-
-       Convergence criterion: There are two modes, residual- and error-based,
-       which can be toggled using \ref SetSurfaceFittingConvergenceBasedOnError.
-
-       (i) Residual based (default): Stop when the norm of the gradient of the
-       TMOP objective reaches the prescribed tolerance. This method is best used
-       with a reasonable value for \ref surf_fit_weight_limit when the
-       adaptive surface fitting scheme is used. See method
-       \ref SetSurfaceFittingWeightLimit.
-
-       (ii) Error based: Stop when the maximum fitting error
-       reaches the user-prescribed threshold, \ref surf_fit_max_err_limit.
-       In this case, \ref surf_fit_weight_limit is ignored during weight
-       adaptation.
-   */
-   ///@{
+   /** @name Methods for adaptive surface fitting weight. (Experimental) */
+   /// Enable/Disable adaptive surface fitting weight.
+   /// The weight is modified after each TMOPNewtonSolver iteration as:
+   /// w_{k+1} = w_{k} * @a surf_fit_scale_factor if relative change in
+   /// max surface fitting error < @a surf_fit_rel_change_threshold.
+   /// The solver terminates if the maximum surface fitting error does
+   /// not sufficiently decrease for @a max_adapt_inc_count consecutive
+   /// solver iterations or if the max error falls below @a surf_fit_max_threshold.
+   void EnableAdaptiveSurfaceFitting()
+   {
+      surf_fit_scale_factor = 10.0;
+      surf_fit_rel_change_threshold = 0.001;
+   }
    void SetAdaptiveSurfaceFittingScalingFactor(real_t factor)
    {
-      MFEM_VERIFY(factor > 1.0, "Scaling factor must be greater than 1.");
       surf_fit_scale_factor = factor;
    }
    void SetAdaptiveSurfaceFittingRelativeChangeThreshold(real_t threshold)
    {
-      surf_fit_err_rel_change_limit = threshold;
+      surf_fit_rel_change_threshold = threshold;
    }
-   /// Used for stopping based on the number of consecutive failed weight
-   /// adaptation iterations.
-   // TODO: Rename to SetMaxNumberofIncrementsForAdaptiveSurfaceFitting
-   // in future.
    void SetMaxNumberofIncrementsForAdaptiveFitting(int count)
    {
-      surf_fit_adapt_count_limit = count;
+      max_adapt_inc_count = count;
    }
-   /// Used for error-based surface fitting termination.
    void SetTerminationWithMaxSurfaceFittingError(real_t max_error)
    {
-      surf_fit_max_err_limit = max_error;
-      surf_fit_converge_error = true;
+      surf_fit_max_threshold = max_error;
    }
-   /// Could be used with both error-based or residual-based convergence.
-   void SetSurfaceFittingMaxErrorLimit(real_t max_error)
-   {
-      surf_fit_max_err_limit = max_error;
-   }
-   /// Used for residual-based surface fitting termination.
-   void SetSurfaceFittingWeightLimit(real_t weight)
-   {
-      surf_fit_weight_limit = weight;
-   }
-   /// Toggle convergence based on residual or error.
-   void SetSurfaceFittingConvergenceBasedOnError(bool mode)
-   {
-      surf_fit_converge_error = mode;
-      if (surf_fit_converge_error)
-      {
-         MFEM_VERIFY(surf_fit_max_err_limit >= 0,
-                     "Fitting error based convergence requires the user to "
-                     "first set the error threshold."
-                     "See SetTerminationWithMaxSurfaceFittingError");
-      }
-   }
-   ///@}
-
-   /// Set minimum determinant enforced during line-search.
    void SetMinimumDeterminantThreshold(real_t threshold)
    {
-      min_detJ_limit = threshold;
+      min_detJ_threshold = threshold;
    }
 
    virtual void Mult(const Vector &b, Vector &x) const

general/array.cpp

@@ -124,7 +124,7 @@ T Array<T>::Sum()
 }
 
 template <class T>
-int Array<T>::IsSorted() const
+int Array<T>::IsSorted()
 {
    T val_prev = operator[](0), val;
    for (int i = 1; i < size; i++)

general/array.hpp

@@ -74,14 +74,10 @@ public:
    inline Array(int asize, MemoryType mt)
       : size(asize) { asize > 0 ? data.New(asize, mt) : data.Reset(mt); }
 
-   /** @brief Creates array using an externally allocated host pointer @a data_
-       to @a asize elements. If @a own_data is true, the array takes ownership
-       of the pointer.
-
-       When @a own_data is true, the pointer @a data_ must be allocated with
-       MemoryType given by MemoryManager::GetHostMemoryType(). */
-   inline Array(T *data_, int asize, bool own_data = false)
-   { data.Wrap(data_, asize, own_data); size = asize; }
+   /** @brief Creates array using an externally allocated pointer @a data_ to
+       @a asize elements. The data pointer will not be deleted by Array. */
+   inline Array(T *data_, int asize)
+   { data.Wrap(data_, asize, false); size = asize; }
 
    /// Copy constructor: deep copy from @a src
    /** This method supports source arrays using any MemoryType. */
@@ -209,14 +205,7 @@ public:
    inline void Copy(Array &copy) const;
 
    /// Make this Array a reference to a pointer.
-   /** When @a own_data is true, the pointer @a data_ must be allocated with
-       MemoryType given by MemoryManager::GetHostMemoryType(). */
-   inline void MakeRef(T *data_, int size_, bool own_data = false);
-
-   /// Make this Array a reference to a pointer.
-   /** When @a own_data is true, the pointer @a data_ must be allocated with
-       MemoryType given by @a mt. */
-   inline void MakeRef(T *data_, int size, MemoryType mt, bool own_data);
+   inline void MakeRef(T *, int);
 
    /// Make this Array a reference to 'master'.
    inline void MakeRef(const Array &master);
@@ -273,7 +262,7 @@ public:
    }
 
    /// Return 1 if the array is sorted from lowest to highest.  Otherwise return 0.
-   int IsSorted() const;
+   int IsSorted();
 
    /// Fill the entries of the array with the cumulative sum of the entries.
    void PartialSum();
@@ -879,19 +868,11 @@ inline void Array<T>::Copy(Array &copy) const
 }
 
 template <class T>
-inline void Array<T>::MakeRef(T *data_, int size_, bool own_data)
+inline void Array<T>::MakeRef(T *p, int s)
 {
    data.Delete();
-   data.Wrap(data_, size_, own_data);
-   size = size_;
-}
-
-template <class T>
-inline void Array<T>::MakeRef(T *data_, int size_, MemoryType mt, bool own_data)
-{
-   data.Delete();
-   data.Wrap(data_, size_, mt, own_data);
-   size = size_;
+   data.Wrap(p, s, false);
+   size = s;
 }
 
 template <class T>

general/arrays_by_name.hpp

@@ -288,3 +288,4 @@ void ArraysByName<T>::Load(std::istream &in)
 }
 
 #endif
+

general/communication.cpp

@@ -275,7 +275,7 @@ void GroupTopology::Save(ostream &os) const
    os << "\ncommunication_groups\n";
    os << "number_of_groups " << NGroups() << "\n\n";
 
-   os << "# number of entities in each group, followed by ranks in group\n";
+   os << "# number of entities in each group, followed by group ids in group\n";
    for (int group_id = 0; group_id < NGroups(); ++group_id)
    {
       int group_size = GetGroupSize(group_id);

general/device.cpp

@@ -14,9 +14,6 @@
 #ifdef MFEM_USE_CEED
 #include "../fem/ceed/interface/util.hpp"
 #endif
-#ifdef MFEM_USE_MPI
-#include "../linalg/hypre.hpp"
-#endif
 
 #include <unordered_map>
 #include <string>
@@ -253,10 +250,6 @@ void Device::Configure(const std::string &device, const int device_id)
 
    // Only '*this' will call the MemoryManager::Destroy() method.
    destroy_mm = true;
-
-#ifdef MFEM_USE_MPI
-   Hypre::InitDevice();
-#endif
 }
 
 // static method

general/forall.hpp

@@ -19,9 +19,6 @@
 #include "device.hpp"
 #include "mem_manager.hpp"
 #include "../linalg/dtensor.hpp"
-#ifdef MFEM_USE_MPI
-#include <_hypre_utilities.h>
-#endif
 
 namespace mfem
 {
@@ -783,63 +780,6 @@ inline void forall_3D_grid(int N, int X, int Y, int Z, int G, lambda &&body)
    ForallWrap<3>(true, N, body, X, Y, Z, G);
 }
 
-#ifdef MFEM_USE_MPI
-
-// Function mfem::hypre_forall_cpu() similar to mfem::forall, but it always
-// executes on the CPU using sequential or OpenMP-parallel execution based on
-// the hypre build time configuration.
-template<typename lambda>
-inline void hypre_forall_cpu(int N, lambda &&body)
-{
-#ifdef HYPRE_USING_OPENMP
-   #pragma omp parallel for HYPRE_SMP_SCHEDULE
-#endif
-   for (int i = 0; i < N; i++) { body(i); }
-}
-
-// Function mfem::hypre_forall_gpu() similar to mfem::forall, but it always
-// executes on the GPU device that hypre was configured with at build time.
-#if defined(HYPRE_USING_GPU)
-template<typename lambda>
-inline void hypre_forall_gpu(int N, lambda &&body)
-{
-#if defined(HYPRE_USING_CUDA)
-   CuWrap1D(N, body);
-#elif defined(HYPRE_USING_HIP)
-   HipWrap1D(N, body);
-#else
-#error Unknown HYPRE GPU backend!
-#endif
-}
-#endif
-
-// Function mfem::hypre_forall() similar to mfem::forall, but it executes on the
-// device, CPU or GPU, that hypre was configured with at build time (when the
-// HYPRE version is < 2.31.0) or at runtime (when HYPRE was configured with GPU
-// support at build time and HYPRE's version is >= 2.31.0). This selection is
-// generally independent of what device was selected in MFEM's runtime
-// configuration.
-template<typename lambda>
-inline void hypre_forall(int N, lambda &&body)
-{
-#if !defined(HYPRE_USING_GPU)
-   hypre_forall_cpu(N, body);
-#elif MFEM_HYPRE_VERSION < 23100
-   hypre_forall_gpu(N, body);
-#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-   if (!HypreUsingGPU())
-   {
-      hypre_forall_cpu(N, body);
-   }
-   else
-   {
-      hypre_forall_gpu(N, body);
-   }
-#endif
-}
-
-#endif // MFEM_USE_MPI
-
 } // namespace mfem
 
 #endif // MFEM_FORALL_HPP

general/mem_manager.cpp

@@ -1154,10 +1154,6 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
    //  dest   d  | h2d   d2d   d2d
    //        hd  | h2h   d2d   d2d
 
-   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
-   MFEM_ASSERT(dst_h_ptr != nullptr, "invalid dst_h_ptr = nullptr");
-   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
-
    const bool dst_on_host =
       (dst_flags & Mem::VALID_HOST) &&
       (!(dst_flags & Mem::VALID_DEVICE) ||
@@ -1233,10 +1229,6 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
                                 size_t bytes, unsigned src_flags)
 {
-   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
-   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
-   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
-
    const bool src_on_host = src_flags & Mem::VALID_HOST;
    if (src_on_host)
    {
@@ -1263,10 +1255,6 @@ void MemoryManager::CopyToHost_(void *dest_h_ptr, const void *src_h_ptr,
 void MemoryManager::CopyFromHost_(void *dest_h_ptr, const void *src_h_ptr,
                                   size_t bytes, unsigned &dest_flags)
 {
-   MFEM_ASSERT(bytes != 0, "this method should not be called with bytes = 0");
-   MFEM_ASSERT(dest_h_ptr != nullptr, "invalid dest_h_ptr = nullptr");
-   MFEM_ASSERT(src_h_ptr != nullptr, "invalid src_h_ptr = nullptr");
-
    const bool dest_on_host = dest_flags & Mem::VALID_HOST;
    if (dest_on_host)
    {

general/mem_manager.hpp

@@ -18,14 +18,8 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
-
 #ifdef MFEM_USE_MPI
-// Enable internal hypre timing routines
-#define HYPRE_TIMING
-#include <HYPRE_utilities.h> // for HYPRE_GetMemoryLocation() and others
-#if (21400 <= MFEM_HYPRE_VERSION) && (MFEM_HYPRE_VERSION < 21900)
-#include <_hypre_utilities.h> // for HYPRE_MEMORY_HOST and others
-#endif
+#include <HYPRE_config.h> // HYPRE_USING_GPU
 #endif
 
 namespace mfem
@@ -875,45 +869,6 @@ public:
 };
 
 
-#ifdef MFEM_USE_MPI
-
-#if MFEM_HYPRE_VERSION < 21400
-#define HYPRE_MEMORY_DEVICE (0)
-#define HYPRE_MEMORY_HOST   (1)
-#endif
-#if MFEM_HYPRE_VERSION < 21900
-typedef int HYPRE_MemoryLocation;
-#endif
-
-/// Return the configured HYPRE_MemoryLocation
-inline HYPRE_MemoryLocation GetHypreMemoryLocation()
-{
-#if !defined(HYPRE_USING_GPU)
-   return HYPRE_MEMORY_HOST;
-#elif MFEM_HYPRE_VERSION < 23100
-   return HYPRE_MEMORY_DEVICE;
-#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-   HYPRE_MemoryLocation loc;
-   HYPRE_GetMemoryLocation(&loc);
-   return loc;
-#endif
-}
-
-/// Return true if HYPRE is configured to use GPU
-inline bool HypreUsingGPU()
-{
-#if !defined(HYPRE_USING_GPU)
-   return false;
-#elif MFEM_HYPRE_VERSION < 23100
-   return true;
-#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-   return GetHypreMemoryLocation() != HYPRE_MEMORY_HOST;
-#endif
-}
-
-#endif // MFEM_USE_MPI
-
-
 // Inline methods
 
 template <typename T>
@@ -1049,12 +1004,10 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
          // If the following condition is true then MemoryManager::Exists()
          // should also be true:
          IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
-#elif MFEM_HYPRE_VERSION < 23100
-         // When HYPRE_USING_GPU is defined and HYPRE < 2.31.0, we always
-         // register the 'base' if the MemoryManager::Exists():
+#else
+         // When HYPRE_USING_GPU is defined we always register the 'base' if
+         // the MemoryManager::Exists():
          MemoryManager::Exists()
-#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-         MemoryManager::Exists() && HypreUsingGPU()
 #endif
       )
       {
@@ -1260,10 +1213,9 @@ template <typename T>
 inline void Memory<T>::CopyFrom(const Memory &src, int size)
 {
    MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
-   if (size <= 0) { return; }
    if (!(flags & Registered) && !(src.flags & Registered))
    {
-      if (h_ptr != src.h_ptr)
+      if (h_ptr != src.h_ptr && size != 0)
       {
          MFEM_ASSERT(h_ptr + size <= src.h_ptr || src.h_ptr + size <= h_ptr,
                      "data overlaps!");
@@ -1281,10 +1233,9 @@ template <typename T>
 inline void Memory<T>::CopyFromHost(const T *src, int size)
 {
    MFEM_VERIFY(capacity>=size, "Incorrect size");
-   if (size <= 0) { return; }
    if (!(flags & Registered))
    {
-      if (h_ptr != src)
+      if (h_ptr != src && size != 0)
       {
          MFEM_ASSERT(h_ptr + size <= src || src + size <= h_ptr,
                      "data overlaps!");
@@ -1301,6 +1252,7 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
 template <typename T>
 inline void Memory<T>::CopyTo(Memory &dest, int size) const
 {
+   MFEM_VERIFY(capacity>=size, "Incorrect size");
    dest.CopyFrom(*this, size);
 }
 
@@ -1308,10 +1260,9 @@ template <typename T>
 inline void Memory<T>::CopyToHost(T *dest, int size) const
 {
    MFEM_VERIFY(capacity>=size, "Incorrect size");
-   if (size <= 0) { return; }
    if (!(flags & Registered))
    {
-      if (h_ptr != dest)
+      if (h_ptr != dest && size != 0)
       {
          MFEM_ASSERT(h_ptr + size <= dest || dest + size <= h_ptr,
                      "data overlaps!");

general/socketstream.cpp

@@ -134,7 +134,7 @@ int socketbuf::open(const char hostname[], int port)
       {
          closesocket(socket_descriptor);
          socket_descriptor = -2;
-         continue;
+         return -1;
       }
 #endif
 
@@ -148,7 +148,7 @@ int socketbuf::open(const char hostname[], int port)
    }
 
    freeaddrinfo(res);
-   return (socket_descriptor < 0) ? -1 : 0;
+   return 0;
 }
 
 int socketbuf::close()

general/table.hpp

@@ -95,7 +95,7 @@ public:
        not called, it returns the number of possible connections established
        by the used constructor. Otherwise, it is exactly the number of
        established connections before calling Finalize(). */
-   inline int Size_of_connections() const { HostReadI(); return I[size]; }
+   inline int Size_of_connections() const { return I[size]; }
 
    /** Returns index of the connection between element i of TYPE I and
        element j of TYPE II. If there is no connection between element i
@@ -207,12 +207,7 @@ template <> inline void Swap<Table>(Table &a, Table &b)
 void Transpose (const Table &A, Table &At, int ncols_A_ = -1);
 Table * Transpose (const Table &A);
 
-///  @brief Transpose an Array<int>.
-///
-/// The array @a A represents a table where each row @a i has exactly one
-/// connection to the column (TYPE II) index specified by @a A[i].
-///
-/// @note The column (TYPE II) indices in each row of @a At will be sorted.
+///  Transpose an Array<int>
 void Transpose(const Array<int> &A, Table &At, int ncols_A_ = -1);
 
 ///  C = A * B  (as boolean matrices)

general/tic_toc.cpp

@@ -400,9 +400,6 @@ inline double StopWatch::SystTime()
 
 StopWatch::StopWatch() : M(new internal::StopWatch) { }
 
-StopWatch::StopWatch(const StopWatch &sw)
-   : M(new internal::StopWatch(*(sw.M))) { }
-
 void StopWatch::Clear()
 {
    M->Clear();

general/tic_toc.hpp

@@ -40,7 +40,6 @@ private:
 public:
    /// Creates a new (stopped) StopWatch object.
    StopWatch();
-   StopWatch(const StopWatch &);
 
    /// Clear the elapsed time on the stopwatch and restart it if it's running.
    void Clear();

linalg/amgxsolver.cpp

@@ -23,10 +23,6 @@
 #include "amgxsolver.hpp"
 #ifdef MFEM_USE_AMGX
 
-#ifdef MFEM_USE_MPI
-#include "../general/communication.hpp"
-#endif
-
 namespace mfem
 {
 

linalg/blockoperator.cpp

@@ -81,7 +81,7 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
       tmp.SetSize(row_offsets[iRow+1] - row_offsets[iRow]);
       for (int jCol=0; jCol < nColBlocks; ++jCol)
       {
-         if (op(iRow,jCol) && coef(iRow,jCol) != 0.)
+         if (op(iRow,jCol))
          {
             op(iRow,jCol)->Mult(xblock.GetBlock(jCol), tmp);
             yblock.GetBlock(iRow).Add(coef(iRow,jCol), tmp);
@@ -112,7 +112,7 @@ void BlockOperator::MultTranspose (const Vector & x, Vector & y) const
       tmp.SetSize(col_offsets[iRow+1] - col_offsets[iRow]);
       for (int jCol=0; jCol < nRowBlocks; ++jCol)
       {
-         if (op(jCol,iRow) && coef(jCol,iRow) != 0.)
+         if (op(jCol,iRow))
          {
             op(jCol,iRow)->MultTranspose(xblock.GetBlock(jCol), tmp);
             yblock.GetBlock(iRow).Add(coef(jCol,iRow), tmp);

linalg/cpardiso.cpp

@@ -1,14 +1,3 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
 #include "cpardiso.hpp"
 #include "hypre.hpp"
 #include <algorithm>

linalg/densemat.cpp

@@ -532,69 +532,6 @@ MatrixInverse *DenseMatrix::Inverse() const
    return new DenseMatrixInverse(*this);
 }
 
-void DenseMatrix::Exponential()
-{
-   MFEM_ASSERT(Height() == Width() && Height() <= 2,
-               "The matrix must be square and "
-               << "of size less than or equal to 2."
-               << "  Height() = " << Height()
-               << ", Width() = " << Width());
-
-   switch (Height())
-   {
-      case 1:
-      {
-         data[0] = std::exp(data[0]);
-         break;
-      }
-      case 2:
-      {
-         /// Formulas from Corollary 2.4 of doi:10.1109/9.233156
-         /// Note typo in the paper, in the prefactor in the equation under (i).
-         const real_t a = data[0];
-         const real_t b = data[1];
-         const real_t c = data[2];
-         const real_t d = data[3];
-         const real_t e = (a - d)*(a - d) + 4*b*c;
-         const real_t f = std::exp((a + d)/2.0);
-         const real_t g = std::sqrt(std::abs(e)) / 2.0;
-
-         if (e == 0)
-         {
-            data[0] = 1.0 + (a - d)/2.0;
-            data[3] = 1.0 - (a - d)/2.0;
-         }
-         else if (e > 0)
-         {
-            data[0] = std::cosh(g) + (a - d)/2 * std::sinh(g) / g;
-            data[1] = b * std::sinh(g) / g;
-            data[2] = c * std::sinh(g) / g;
-            data[3] = std::cosh(g) - (a - d)/2 * std::sinh(g) / g;
-         }
-         else
-         {
-            data[0] = std::cos(g) + (a - d)/2 * std::sin(g) / g;
-            data[1] = b * std::sin(g) / g;
-            data[2] = c * std::sin(g) / g;
-            data[3] = std::cos(g) - (a - d)/2 * std::sin(g) / g;
-         }
-         for (int i = 0; i < 4; i++)
-         {
-            data[i] *= f;
-         }
-         break;
-      }
-      case 3:
-      {
-         MFEM_ABORT("3x3 matrices are not currently supported");
-      }
-      default:
-      {
-         MFEM_ABORT("Only 1x1 and 2x2 matrices are currently supported");
-      }
-   }
-}
-
 real_t DenseMatrix::Det() const
 {
    MFEM_ASSERT(Height() == Width() && Height() > 0,
@@ -3280,93 +3217,6 @@ void MultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB)
 #endif
 }
 
-void AddMultAtB(const DenseMatrix &A, const DenseMatrix &B,
-                DenseMatrix &AtB)
-{
-   MFEM_ASSERT(AtB.Height() == A.Width() && AtB.Width() == B.Width() &&
-               A.Height() == B.Height(), "incompatible dimensions");
-
-#ifdef MFEM_USE_LAPACK
-   static char transa = 'T', transb = 'N';
-   static real_t alpha = 1.0, beta = 1.0;
-   int m = A.Width(), n = B.Width(), k = A.Height();
-
-#ifdef MFEM_USE_SINGLE
-   sgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
-#elif defined MFEM_USE_DOUBLE
-   dgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
-#endif
-          B.Data(), &k, &beta, AtB.Data(), &m);
-#else
-   const int ah = A.Height();
-   const int aw = A.Width();
-   const int bw = B.Width();
-   const real_t *ad = A.Data();
-   const real_t *bd = B.Data();
-   real_t *cd = AtB.Data();
-
-   for (int j = 0; j < bw; j++)
-   {
-      const real_t *ap = ad;
-      for (int i = 0; i < aw; i++)
-      {
-         real_t d = 0.0;
-         for (int k = 0; k < ah; k++)
-         {
-            d += ap[k] * bd[k];
-         }
-         *(cd++) += d;
-         ap += ah;
-      }
-      bd += ah;
-   }
-#endif
-}
-
-void AddMult_a_AtB(real_t a, const DenseMatrix &A, const DenseMatrix &B,
-                   DenseMatrix &AtB)
-{
-   MFEM_ASSERT(AtB.Height() == A.Width() && AtB.Width() == B.Width() &&
-               A.Height() == B.Height(), "incompatible dimensions");
-
-#ifdef MFEM_USE_LAPACK
-   static char transa = 'T', transb = 'N';
-   real_t alpha = a;
-   static real_t beta = 1.0;
-   int m = A.Width(), n = B.Width(), k = A.Height();
-
-#ifdef MFEM_USE_SINGLE
-   sgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
-#elif defined MFEM_USE_DOUBLE
-   dgemm_(&transa, &transb, &m, &n, &k, &alpha, A.Data(), &k,
-#endif
-          B.Data(), &k, &beta, AtB.Data(), &m);
-#else
-   const int ah = A.Height();
-   const int aw = A.Width();
-   const int bw = B.Width();
-   const real_t *ad = A.Data();
-   const real_t *bd = B.Data();
-   real_t *cd = AtB.Data();
-
-   for (int j = 0; j < bw; j++)
-   {
-      const real_t *ap = ad;
-      for (int i = 0; i < aw; i++)
-      {
-         real_t d = 0.0;
-         for (int k = 0; k < ah; k++)
-         {
-            d += ap[k] * bd[k];
-         }
-         *(cd++) += a * d;
-         ap += ah;
-      }
-      bd += ah;
-   }
-#endif
-}
-
 void AddMult_a_AAt(real_t a, const DenseMatrix &A, DenseMatrix &AAt)
 {
    real_t d;

linalg/densemat.hpp

@@ -207,10 +207,6 @@ public:
    /// Replaces the current matrix with its square root inverse
    void SquareRootInverse();
 
-   /// Replaces the current matrix with its exponential
-   /// (currently only supports 2x2 matrices)
-   void Exponential();
-
    /// Calculates the determinant of the matrix
    /// (optimized for 2x2, 3x3, and 4x4 matrices)
    real_t Det() const;
@@ -584,13 +580,6 @@ void AddMult_a_ABt(real_t a, const DenseMatrix &A, const DenseMatrix &B,
 /// Multiply the transpose of a matrix A with a matrix B:   At*B
 void MultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB);
 
-/// AtB += A^t * B
-void AddMultAtB(const DenseMatrix &A, const DenseMatrix &B, DenseMatrix &AtB);
-
-/// AtB += a * A^t * B
-void AddMult_a_AtB(real_t a, const DenseMatrix &A, const DenseMatrix &B,
-                   DenseMatrix &AtB);
-
 /// AAt += a * A * A^t
 void AddMult_a_AAt(real_t a, const DenseMatrix &A, DenseMatrix &AAt);
 

linalg/hypre.hpp

@@ -16,19 +16,17 @@
 
 #ifdef MFEM_USE_MPI
 
-#include "../general/globals.hpp"
-#include "sparsemat.hpp"
-#include "hypre_parcsr.hpp"
 #include <mpi.h>
 
 // Enable internal hypre timing routines
 #define HYPRE_TIMING
 
 // hypre header files
-#include <seq_mv.h>
-#include <temp_multivector.h>
-#include <_hypre_parcsr_mv.h>
-#include <_hypre_parcsr_ls.h>
+#include "seq_mv.h"
+#include "_hypre_parcsr_mv.h"
+#include "_hypre_parcsr_ls.h"
+#include "temp_multivector.h"
+#include "../general/globals.hpp"
 
 #ifdef HYPRE_COMPLEX
 #error "MFEM does not work with HYPRE's complex numbers support"
@@ -53,6 +51,22 @@
 #error "MFEM_USE_HIP=YES is required when HYPRE is built with HIP!"
 #endif
 
+// MFEM_HYPRE_FORALL is a macro similar to mfem::forall, but it executes on the
+// device that hypre was configured with (no matter what device was selected
+// in MFEM's runtime configuration).
+#if defined(HYPRE_USING_CUDA)
+#define MFEM_HYPRE_FORALL(i, N,...) CuWrap1D(N, [=] MFEM_DEVICE      \
+                                       (int i) {__VA_ARGS__})
+#elif defined(HYPRE_USING_HIP)
+#define MFEM_HYPRE_FORALL(i, N,...) HipWrap1D(N, [=] MFEM_DEVICE     \
+                                        (int i) {__VA_ARGS__})
+#else
+#define MFEM_HYPRE_FORALL(i, N,...) for (int i = 0; i < N; i++) { __VA_ARGS__ }
+#endif
+
+#include "sparsemat.hpp"
+#include "hypre_parcsr.hpp"
+
 namespace mfem
 {
 
@@ -73,22 +87,6 @@ public:
    /// Calling HYPRE_Finalize() manually is not compatible with this class.
    static void Init() { Instance(); }
 
-   /// @brief Configure HYPRE's compute and memory policy.
-   ///
-   /// By default HYPRE will be configured with the same policy as MFEM unless
-   /// `Hypre::configure_runtime_policy_from_mfem` is false, in which case
-   /// HYPRE's default will be used; if HYPRE is built for the GPU and the
-   /// aforementioned variable is false then HYPRE will use the GPU even if MFEM
-   /// is not.
-   ///
-   /// This function is no-op if HYPRE is built without GPU support or the HYPRE
-   /// version is less than 2.31.0.
-   ///
-   /// This function is NOT called by Init(). Instead it is called by
-   /// Device::Configure() (when MFEM_USE_MPI=YES) after the MFEM device
-   /// configuration is complete.
-   static void InitDevice();
-
    /// @brief Finalize hypre (called automatically at program exit if
    /// Hypre::Init() has been called).
    ///
@@ -96,13 +94,6 @@ public:
    /// called manually to more precisely control when hypre is finalized.
    static void Finalize();
 
-   /// @brief Use MFEM's device policy to configure HYPRE's device policy, true
-   /// by default. This variable is used by InitDevice().
-   ///
-   /// This value is not used if HYPRE is build without GPU support or the HYPRE
-   /// version is less than 2.31.0.
-   static bool configure_runtime_policy_from_mfem;
-
 private:
    /// Calls HYPRE_Init() when the singleton is constructed.
    Hypre();
@@ -151,28 +142,15 @@ inline int to_int(HYPRE_Int i)
 
 
 /// The MemoryClass used by Hypre objects.
-inline MemoryClass GetHypreMemoryClass()
+inline constexpr MemoryClass GetHypreMemoryClass()
 {
 #if !defined(HYPRE_USING_GPU)
    return MemoryClass::HOST;
-#elif MFEM_HYPRE_VERSION < 23100
-#if defined(HYPRE_USING_UNIFIED_MEMORY)
+#elif defined(HYPRE_USING_UNIFIED_MEMORY)
    return MemoryClass::MANAGED;
 #else
    return MemoryClass::DEVICE;
 #endif
-#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-   if (GetHypreMemoryLocation() == HYPRE_MEMORY_HOST)
-   {
-      return MemoryClass::HOST;
-   }
-   // Return the actual memory location, see hypre_GetActualMemLocation():
-#if defined(HYPRE_USING_UNIFIED_MEMORY)
-   return MemoryClass::MANAGED;
-#else
-   return MemoryClass::DEVICE;
-#endif
-#endif
 }
 
 /// The MemoryType used by MFEM when allocating arrays for Hypre objects.
@@ -180,27 +158,13 @@ inline MemoryType GetHypreMemoryType()
 {
 #if !defined(HYPRE_USING_GPU)
    return Device::GetHostMemoryType();
-#elif MFEM_HYPRE_VERSION < 23100
-#if defined(HYPRE_USING_UNIFIED_MEMORY)
+#elif defined(HYPRE_USING_UNIFIED_MEMORY)
    return MemoryType::MANAGED;
 #else
    return MemoryType::DEVICE;
 #endif
-#else // HYPRE_USING_GPU is defined and MFEM_HYPRE_VERSION >= 23100
-   if (GetHypreMemoryLocation() == HYPRE_MEMORY_HOST)
-   {
-      return Device::GetHostMemoryType();
-   }
-   // Return the actual memory location, see hypre_GetActualMemLocation():
-#if defined(HYPRE_USING_UNIFIED_MEMORY)
-   return MemoryType::MANAGED;
-#else
-   return MemoryType::DEVICE;
-#endif
-#endif
 }
 
-
 /// Wrapper for hypre's parallel vector class
 class HypreParVector : public Vector
 {
@@ -364,10 +328,10 @@ public:
    HYPRE_Int Randomize(HYPRE_Int seed);
 
    /// Prints the locally owned rows in parallel
-   void Print(const std::string &fname) const;
+   void Print(const char *fname) const;
 
    /// Reads a HypreParVector from files saved with HypreParVector::Print
-   void Read(MPI_Comm comm, const std::string &fname);
+   void Read(MPI_Comm comm, const char *fname);
 
    /// Calls hypre's destroy function
    ~HypreParVector();
@@ -919,14 +883,12 @@ public:
    const Memory<HYPRE_Int> &GetDiagMemoryJ() const { return mem_diag.J; }
    const Memory<real_t> &GetDiagMemoryData() const { return mem_diag.data; }
 
-   /// @brief Prints the locally owned rows in parallel. The resulting files can
-   /// be read with Read_IJMatrix().
-   void Print(const std::string &fname, HYPRE_Int offi = 0,
-              HYPRE_Int offj = 0) const;
+   /// Prints the locally owned rows in parallel
+   void Print(const char *fname, HYPRE_Int offi = 0, HYPRE_Int offj = 0) const;
    /// Reads the matrix from a file
-   void Read(MPI_Comm comm, const std::string &fname);
+   void Read(MPI_Comm comm, const char *fname);
    /// Read a matrix saved as a HYPRE_IJMatrix
-   void Read_IJMatrix(MPI_Comm comm, const std::string &fname);
+   void Read_IJMatrix(MPI_Comm comm, const char *fname);
 
    /// Print information about the hypre_ParCSRCommPkg of the HypreParMatrix.
    void PrintCommPkg(std::ostream &out = mfem::out) const;
@@ -1075,40 +1037,29 @@ protected:
    bool A_is_symmetric;
 
 public:
-   /// HYPRE smoother types
-   enum Type
-   {
-      Jacobi = 0,       ///< Jacobi
-      l1Jacobi = 1,     ///< l1-scaled Jacobi
-      l1GS = 2,         ///< l1-scaled block Gauss-Seidel/SSOR
-      l1GStr = 4,       ///< truncated l1-scaled block Gauss-Seidel/SSOR
-      lumpedJacobi = 5, ///< lumped Jacobi
-      GS = 6,           ///< Gauss-Seidel
-      OPFS = 10,        /**< On-processor forward solve for matrix w/ triangular
-                             structure */
-      Chebyshev = 16,   ///< Chebyshev
-      Taubin = 1001,    ///< Taubin polynomial smoother
-      FIR = 1002        ///< FIR polynomial smoother
-   };
-
-   /// @deprecated Use DefaultType() instead
+   /** Hypre smoother types:
+       0    = Jacobi
+       1    = l1-scaled Jacobi
+       2    = l1-scaled block Gauss-Seidel/SSOR
+       4    = truncated l1-scaled block Gauss-Seidel/SSOR
+       5    = lumped Jacobi
+       6    = Gauss-Seidel
+       10   = On-processor forward solve for matrix w/ triangular structure
+       16   = Chebyshev
+       1001 = Taubin polynomial smoother
+       1002 = FIR polynomial smoother. */
+   enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, l1GStr = 4, lumpedJacobi = 5,
+               GS = 6, OPFS = 10, Chebyshev = 16, Taubin = 1001, FIR = 1002
+             };
 #if !defined(HYPRE_USING_GPU)
-   MFEM_DEPRECATED static constexpr Type default_type = l1GS;
+   static constexpr Type default_type = l1GS;
 #else
-   MFEM_DEPRECATED static constexpr Type default_type = l1Jacobi;
+   static constexpr Type default_type = l1Jacobi;
 #endif
 
-   /** @brief Default value for the smoother type used by the constructors:
-       Type::l1GS when HYPRE is running on CPU and Type::l1Jacobi when HYPRE is
-       running on GPU. */
-   static Type DefaultType()
-   {
-      return HypreUsingGPU() ? l1Jacobi : l1GS;
-   }
-
    HypreSmoother();
 
-   HypreSmoother(const HypreParMatrix &A_, int type = DefaultType(),
+   HypreSmoother(const HypreParMatrix &A_, int type = default_type,
                  int relax_times = 1, real_t relax_weight = 1.0,
                  real_t omega = 1.0, int poly_order = 2,
                  real_t poly_fraction = .3, int eig_est_cg_iter = 10);
@@ -1523,53 +1474,12 @@ public:
 
    virtual void SetOperator(const Operator &op);
 
-   /// Set the threshold and levels parameters
-   /** The accuracy and cost of ParaSails are parametrized by the real
-    * @a thresh and integer @a nlevels parameters (0<=thresh<=1,  0<=nlevels).
-    * Lower values of @a thresh and higher values of @a nlevels lead to
-    * more accurate, but more expensive preconditioners. More accurate
-    * preconditioners are also more expensive per iteration. The default
-    * values are @a thresh = 0.1 and @a nlevels = 1.
-    */
-   void SetParams(real_t thresh, int nlevels);
-
-   /// Set the filter parameter
-   /** The filter parameter is used to drop small nonzeros in the preconditioner,
-    * to reduce the cost of applying the preconditioner. Values from 0.055
-    * to 0.1 are recommended. The default value is 0.1.
-    */
+   void SetParams(real_t threshold, int max_levels);
    void SetFilter(real_t filter);
-
-   /// Set symmetry parameter
-   /** The recognized options are:
-    *  0 = nonsymmetric and/or indefinite problem, and nonsymmetric preconditioner
-    *  1 = SPD problem, and SPD (factored) preconditioner
-    *  2 = nonsymmetric, definite problem, and SPD (factored) preconditioner
-    */
-   void SetSymmetry(int sym);
-
-   /// Set the load balance parameter
-   /** A zero value indicates that no load balance is attempted; a value
-    * of unity indicates that perfect load balance will be attempted. The
-    * recommended value is 0.9 to balance the overhead of data exchanges
-    * for load balancing. No load balancing is needed if the preconditioner
-    * is very sparse and fast to construct. The default value is 0.
-    */
    void SetLoadBal(real_t loadbal);
-
-   /// Set the pattern reuse parameter
-   /** A nonzero value indicates that the pattern of the preconditioner
-    * should be reused for subsequent constructions of the proconditioner.
-    * A zero value inicates that the peconditioner should be constructed
-    * from scratch. The default value is 0.
-    */
    void SetReuse(int reuse);
-
-   /// Set the logging parameter
-   /** A nonzero value prints statistics of the setup procedure to stdout.
-    * The default value of this parameter is 1.
-    */
    void SetLogging(int logging);
+   void SetSymmetry(int sym);
 
    /// The typecast to HYPRE_Solver returns the internal sai_precond
    virtual operator HYPRE_Solver() const { return sai_precond; }

linalg/hypre_parcsr.hpp

@@ -16,10 +16,11 @@
 
 #ifdef MFEM_USE_MPI
 
-#include "../general/mem_manager.hpp"
+// Enable internal hypre timing routines
+#define HYPRE_TIMING
 
-// hypre header files
-#include <_hypre_parcsr_mv.h>
+#include "../general/mem_manager.hpp"
+#include "_hypre_parcsr_mv.h"
 
 // Older hypre versions do not define HYPRE_BigInt and HYPRE_MPI_BIG_INT, so we
 // define them here for backward compatibility.
@@ -45,10 +46,10 @@ typedef HYPRE_Int HYPRE_BigInt;
 #else // MFEM_HYPRE_VERSION >= 21400
 
 #define mfem_hypre_TAlloc(type, size) \
-   hypre_TAlloc(type, size, mfem::GetHypreMemoryLocation())
+   hypre_TAlloc(type, size, HYPRE_MEMORY_DEVICE)
 #define mfem_hypre_CTAlloc(type, size) \
-   hypre_CTAlloc(type, size, mfem::GetHypreMemoryLocation())
-#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, mfem::GetHypreMemoryLocation())
+   hypre_CTAlloc(type, size, HYPRE_MEMORY_DEVICE)
+#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, HYPRE_MEMORY_DEVICE)
 
 #define mfem_hypre_TAlloc_host(type, size) \
    hypre_TAlloc(type, size, HYPRE_MEMORY_HOST)

linalg/operator.hpp

@@ -309,51 +309,18 @@ public:
 
 
 /// Base abstract class for first order time dependent operators.
-/** Operator of the form: (u,t) -> k(u,t), where k generally solves the
-    algebraic equation F(u,k,t) = G(u,t). The functions F and G represent the
-    _implicit_ and _explicit_ parts of the operator, respectively.
-
-    A common use for this class is representing a differential algebraic
-    equation of the form $ F(y,\frac{dy}{dt},t) = G(y,t) $.
-
-    For example, consider an ordinary differential equation of the form
-    $ M \frac{dy}{dt} = g(y,t) $. There are various ways of expressing this ODE
-    as a TimeDependentOperator depending on the choices for F and G. Here are
-    some common choices:
-
-      1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t),
-      2. F(u,k,t) = M k and G(u,t) = g(u,t),
-      3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.
-
-    Note that depending on the ODE solver, some of the above choices may be
-    preferable to the others.
-*/
+/** Operator of the form: (x,t) -> f(x,t), where k = f(x,t) generally solves the
+    algebraic equation F(x,k,t) = G(x,t). The functions F and G represent the
+    _implicit_ and _explicit_ parts of the operator, respectively. For explicit
+    operators, F(x,k,t) = k, so f(x,t) = G(x,t). */
 class TimeDependentOperator : public Operator
 {
 public:
-   /// Enum used to describe the form of the time-dependent operator.
-   /** The type should be set by classes derived from TimeDependentOperator to
-       describe the form, in terms of the functions F and G, used by the
-       specific derived class. This information can be queried by classes or
-       functions (like time stepping algorithms) to make choices about the
-       algorithm to use, or to ensure that the TimeDependentOperator uses the
-       form expected by the class/function.
-
-       For example, assume that a derived class is implementing the ODE
-       $M \frac{dy}{dt} = g(y,t)$ and chooses to define $F(u,k,t) = M k$ and
-       $G(u,t) = g(u,t)$. Then it cannot use type EXPLICIT, unless $M = I$, or
-       type HOMOGENEOUS, unless $g(u,t) = 0$. If, on the other hand, the derived
-       class chooses to define $F(u,k,t) = k$ and $G(u,t) = M^{-1} g(y,t)$, then
-       the natural choice is to set the type to EXPLICIT, even though setting it
-       to IMPLICIT is also not wrong -- doing so will simply fail to inform
-       methods that query this information that it uses a more specific
-       implementation, EXPLICIT, that may allow the use of algorithms that
-       support only the EXPLICIT type. */
    enum Type
    {
-      EXPLICIT,   ///< This type assumes F(u,k,t) = k.
+      EXPLICIT,   ///< This type assumes F(x,k,t) = k, i.e. k = f(x,t) = G(x,t).
       IMPLICIT,   ///< This is the most general type, no assumptions on F and G.
-      HOMOGENEOUS ///< This type assumes that G(u,t) = 0.
+      HOMOGENEOUS ///< This type assumes that G(x,t) = 0.
    };
 
    /// Evaluation mode. See SetEvalMode() for details.
@@ -361,30 +328,29 @@ public:
    {
       /** Normal evaluation. */
       NORMAL,
-      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
-          first term, k1. */
+      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
+          first term, f1. */
       ADDITIVE_TERM_1,
-      /** Assuming additive split, k(u,t) = k1(u,t) + k2(u,t), evaluate the
-          second term, k2. */
+      /** Assuming additive split, f(x,t) = f1(x,t) + f2(x,t), evaluate the
+          second term, f2. */
       ADDITIVE_TERM_2
    };
 
 protected:
    real_t t;  ///< Current time.
-   Type type; /**< @brief Describes the form of the TimeDependentOperator, see
-                   the documentation of #Type. */
+   Type type; ///< Describes the form of the TimeDependentOperator.
    EvalMode eval_mode; ///< Current evaluation mode.
 
 public:
-   /** @brief Construct a "square" TimeDependentOperator (u,t) -> k(u,t), where
-       u and k have the same dimension @a n. */
+   /** @brief Construct a "square" TimeDependentOperator y = f(x,t), where x and
+       y have the same dimension @a n. */
    explicit TimeDependentOperator(int n = 0, real_t t_ = 0.0,
                                   Type type_ = EXPLICIT)
       : Operator(n) { t = t_; type = type_; eval_mode = NORMAL; }
 
-   /** @brief Construct a TimeDependentOperator (u,t) -> k(u,t), where u and k
-       have dimensions @a w and @a h, respectively. */
-   TimeDependentOperator(int h, int w, double t_ = 0.0, Type type_ = EXPLICIT)
+   /** @brief Construct a TimeDependentOperator y = f(x,t), where x and y have
+       dimensions @a w and @a h, respectively. */
+   TimeDependentOperator(int h, int w, real_t t_ = 0.0, Type type_ = EXPLICIT)
       : Operator(h, w) { t = t_; type = type_; eval_mode = NORMAL; }
 
    /// Read the currently set time.
@@ -407,7 +373,7 @@ public:
    /** The evaluation mode is a switch that allows time-stepping methods to
        request evaluation of separate components/terms of the time-dependent
        operator. For example, IMEX methods typically assume additive split of
-       the operator: k(u,t) = k1(u,t) + k2(u,t) and they rely on the ability to
+       the operator: f(x,t) = f1(x,t) + f2(x,t) and they rely on the ability to
        evaluate the two terms separately.
 
        Generally, setting the evaluation mode should affect the behavior of all
@@ -418,104 +384,62 @@ public:
    { eval_mode = new_eval_mode; }
 
    /** @brief Perform the action of the explicit part of the operator, G:
-       @a v = G(@a u, t) where t is the current time.
+       @a y = G(@a x, t) where t is the current time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual. */
-   virtual void ExplicitMult(const Vector &u, Vector &v) const;
+   virtual void ExplicitMult(const Vector &x, Vector &y) const;
 
    /** @brief Perform the action of the implicit part of the operator, F:
-       @a v = F(@a u, @a k, t) where t is the current time.
+       @a y = F(@a x, @a k, t) where t is the current time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual.*/
-   virtual void ImplicitMult(const Vector &u, const Vector &k, Vector &v) const;
+   virtual void ImplicitMult(const Vector &x, const Vector &k, Vector &y) const;
 
-   /** @brief Perform the action of the operator (u,t) -> k(u,t) where t is the
-       current time set by SetTime() and @a k satisfies
-       F(@a u, @a k, t) = G(@a u, t).
+   /** @brief Perform the action of the operator: @a y = k = f(@a x, t), where
+       k solves the algebraic equation F(@a x, k, t) = G(@a x, t) and t is the
+       current time. */
+   virtual void Mult(const Vector &x, Vector &y) const;
 
-       For solving an ordinary differential equation of the form
-       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
-       various ways, e.g.:
+   /** @brief Solve the equation: @a k = f(@a x + @a dt @a k, t), for the
+       unknown @a k at the current time t.
 
-         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
-         2. F(u,k,t) = M k and G(u,t) = g(u,t)
-         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0.
+       For general F and G, the equation for @a k becomes:
+       F(@a x + @a dt @a k, @a k, t) = G(@a x + @a dt @a k, t).
 
-       Regardless of the choice of F and G, this function should always compute
-       @a k = inv(M) g(@a u, t). */
-   virtual void Mult(const Vector &u, Vector &v) const override;
+       The input vector @a x corresponds to time index (or cycle) n, while the
+       currently set time, #t, and the result vector @a k correspond to time
+       index n+1. The time step @a dt corresponds to the time interval between
+       cycles n and n+1.
 
-   /** @brief Solve for the unknown @a k, at the current time t, the following
-       equation:
-       F(@a u + @a gamma @a k, @a k, t) = G(@a u + @a gamma @a k, t).
-
-       For solving an ordinary differential equation of the form
-       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined in
-       various ways, e.g.:
-
-         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
-         2. F(u,k,t) = M k and G(u,t) = g(u,t)
-         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
-
-       Regardless of the choice of F and G, this function should solve for @a k
-       in M @a k = g(@a u + @a gamma @a k, t).
-
-       To see how @a k can be useful, consider the backward Euler method defined
-       by $ y(t + \Delta t) = y(t) + \Delta t k_0 $ where
-       $ M k_0 = g \big( y(t) + \Delta t k_0, t + \Delta t \big) $. A backward
-       Euler integrator can use @a k from this function for $k_0$, with the call
-       using @a u set to $ y(t) $, @a gamma set to $ \Delta t$, and time set to
-       $t + \Delta t$. See class BackwardEulerSolver.
-
-       Generalizing further, consider a diagonally implicit Runge-Kutta (DIRK)
-       method defined by
-       $ y(t + \Delta t) = y(t) + \Delta t \sum_{i=1}^s b_i k_i $ where
-       $ M k_i = g \big( y(t) + \Delta t \sum_{j=1}^i a_{ij} k_j,
-                         t + c_i \Delta t \big) $.
-       A DIRK integrator can use @a k from this function, with @a u set to
-       $ y(t) + \Delta t \sum_{j=1}^{i-1} a_{ij} k_j $ and @a gamma set to
-       $ a_{ii} \Delta t $, for $ k_i $. For example, see class SDIRK33Solver.
+       This method allows for the abstract implementation of some time
+       integration methods, including diagonal implicit Runge-Kutta (DIRK)
+       methods and the backward Euler method in particular.
 
        If not re-implemented, this method simply generates an error. */
-   virtual void ImplicitSolve(const real_t gamma, const Vector &u, Vector &k);
+   virtual void ImplicitSolve(const real_t dt, const Vector &x, Vector &k);
 
-   /** @brief Return an Operator representing (dF/dk @a shift + dF/du) at the
-       given @a u, @a k, and the currently set time.
+   /** @brief Return an Operator representing (dF/dk @a shift + dF/dx) at the
+       given @a x, @a k, and the currently set time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual. */
-   virtual Operator& GetImplicitGradient(const Vector &u, const Vector &k,
+   virtual Operator& GetImplicitGradient(const Vector &x, const Vector &k,
                                          real_t shift) const;
 
-   /** @brief Return an Operator representing dG/du at the given point @a u and
+   /** @brief Return an Operator representing dG/dx at the given point @a x and
        the currently set time.
 
        Presently, this method is used by some PETSc ODE solvers, for more
        details, see the PETSc Manual. */
-   virtual Operator& GetExplicitGradient(const Vector &u) const;
+   virtual Operator& GetExplicitGradient(const Vector &x) const;
 
-   /** @brief Setup a linear system as needed by some SUNDIALS ODE solvers.
+   /** @brief Setup the ODE linear system $ A(x,t) = (I - gamma J) $ or
+       $ A = (M - gamma J) $, where $ J(x,t) = \frac{df}{dt(x,t)} $.
 
-       For solving an ordinary differential equation of the form
-       $ M \frac{dy}{dt} = g(y,t) $, recall that F and G can be defined as one
-       of the following:
-
-         1. F(u,k,t) = k and G(u,t) = inv(M) g(u,t)
-         2. F(u,k,t) = M k and G(u,t) = g(u,t)
-         3. F(u,k,t) = M k - g(u,t) and G(u,t) = 0
-
-       This function performs setup to solve $ A x = b $ where A is either
-
-         1. A(@a y,t) = I - @a gamma inv(M) J(@a y,t)
-         2. A(@a y,t) = M - @a gamma J(@a y,t)
-         3. A(@a y,t) = M - @a gamma J(@a y,t)
-
-       with J = dg/dy (or a reasonable approximation thereof).
-
-       @param[in]  y     The state at which A(@a y,t) should be evaluated.
-       @param[in]  v     The value of inv(M) g(y,t) for 1 or g(y,t) for 2 & 3.
+       @param[in]  x     The state at which $A(x,t)$ should be evaluated.
+       @param[in]  fx    The current value of the ODE rhs function, $f(x,t)$.
        @param[in]  jok   Flag indicating if the Jacobian should be updated.
        @param[out] jcur  Flag to signal if the Jacobian was updated.
        @param[in]  gamma The scaled time step value.
@@ -524,10 +448,10 @@ public:
 
        Presently, this method is used by SUNDIALS ODE solvers, for more
        details, see the SUNDIALS User Guides. */
-   virtual int SUNImplicitSetup(const Vector &y, const Vector &v,
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
                                 int jok, int *jcur, real_t gamma);
 
-   /** @brief Solve the ODE linear system A @a x = @a b, where A is defined by
+   /** @brief Solve the ODE linear system $ A x = b $ as setup by
        the method SUNImplicitSetup().
 
        @param[in]      b   The linear system right-hand side.
@@ -540,8 +464,7 @@ public:
        details, see the SUNDIALS User Guides. */
    virtual int SUNImplicitSolve(const Vector &b, Vector &x, real_t tol);
 
-   /** @brief Setup the mass matrix in the ODE system
-       $ M \frac{dy}{dt} = g(y,t) $ .
+   /** @brief Setup the mass matrix in the ODE system $ M y' = f(y,t) $ .
 
        If not re-implemented, this method simply generates an error.
 
@@ -549,8 +472,8 @@ public:
        details, see the ARKode User Guide. */
    virtual int SUNMassSetup();
 
-   /** @brief Solve the mass matrix linear system  M @a x = @a b, where M is
-       defined by the method SUNMassSetup().
+   /** @brief Solve the mass matrix linear system $ M x = b $
+       as setup by the method SUNMassSetup().
 
        @param[in]      b   The linear system right-hand side.
        @param[in,out]  x   On input, the initial guess. On output, the solution.
@@ -562,8 +485,7 @@ public:
        details, see the ARKode User Guide. */
    virtual int SUNMassSolve(const Vector &b, Vector &x, real_t tol);
 
-   /** @brief Compute the mass matrix-vector product @a v = M @a x, where M is
-       defined by the method SUNMassSetup().
+   /** @brief Compute the mass matrix-vector product $ v = M x $ .
 
        @param[in]   x The vector to multiply.
        @param[out]  v The result of the matrix-vector product.

linalg/solvers.cpp

@@ -1188,8 +1188,6 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
 
    final_norm = std::max(rel_tol*beta, abs_tol);
 
-   converged = false;
-
    if (beta <= final_norm)
    {
       final_norm = beta;
@@ -1305,6 +1303,8 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
       MFEM_ASSERT(IsFinite(beta), "beta = " << beta);
       if (beta <= final_norm)
       {
+         final_norm = beta;
+         final_iter = j;
          converged = true;
 
          break;
@@ -1317,9 +1317,7 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
       if (v[i]) { delete v[i]; }
       if (z[i]) { delete z[i]; }
    }
-
-   final_norm = beta;
-   final_iter = converged ? j : max_iter;
+   converged = false;
 
    // Note: j is off by one when we arrive here
    if (!print_options.iterations && print_options.first_and_last)
@@ -1330,7 +1328,7 @@ void FGMRESSolver::Mult(const Vector &b, Vector &x) const
    }
    if (print_options.summary || (print_options.warnings && !converged))
    {
-      mfem::out << "FGMRES: Number of iterations: " << final_iter << '\n';
+      mfem::out << "FGMRES: Number of iterations: " << j-1 << '\n';
    }
    if (print_options.warnings && !converged)
    {

linalg/symmat.cpp

@@ -58,20 +58,6 @@ DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(real_t c)
    return *this;
 }
 
-DenseSymmetricMatrix &DenseSymmetricMatrix::operator=(const DenseSymmetricMatrix
-                                                      &m)
-{
-   SetSize(m.height);
-
-   const int hw = m.GetStoredSize();
-   for (int i = 0; i < hw; i++)
-   {
-      data[i] = m.data[i];
-   }
-
-   return *this;
-}
-
 real_t &DenseSymmetricMatrix::Elem(int i, int j)
 {
    return (*this)(i,j);
@@ -103,6 +89,11 @@ MatrixInverse *DenseSymmetricMatrix::Inverse() const
    return nullptr;
 }
 
+void DenseSymmetricMatrix::Print (std::ostream & os, int width_) const
+{
+   mfem_error("DenseSymmetricMatrix::Print() not implemented!");
+}
+
 DenseSymmetricMatrix::~DenseSymmetricMatrix()
 {
    data.Delete();