Merge branch 'fusion-ldrd-dev' into epic-dev

Update version numbers to 4.3.1 -- a new development version

.gitignore

@@ -252,6 +252,7 @@ miniapps/shifted/ParaViewDistance
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
+miniapps/shifted/ParaViewDiffusion
 
 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -296,6 +297,7 @@ tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
+tests/unit/debug_device_tests
 
 # Test script output
 tests/scripts/*.err

.gitlab-ci.yml

@@ -51,9 +51,9 @@ variables:
   MFEM_DATA_REPO: https://github.com/mfem/data.git
   ARTIFACTS_DIR: artifacts
 
-# The pipeline is divided into stages. Usually, these are also synchronization
-# points, however, we use "needs" keyword to express the DAG of jobs for more
-# efficiency.
+# The pipeline is divided into stages. Usually, jobs in a given stage wait for
+# the preceding stages to complete before to start. However, we sometimes use
+# the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
@@ -87,7 +87,6 @@ setup:
   script:
     - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
     - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
-  needs: []
 
 # The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
 # It prepares a pipeline-wide working directory downloading/updating external
@@ -95,6 +94,7 @@ setup:
 # are now using unique directories so repo are never shared with another
 # pipeline. This is not memory efficient (we keep a lot of data), hence this
 # reminder.
+# Note: This job can start immediately.
 setup_baseline:
   tags:
     - shell
@@ -125,10 +125,10 @@ setup_baseline:
   script:
     - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
 
-# Lassen uses a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-# We use pdebug queue on lassen to speed-up the allocation.
-# However this would not be scalable to multiple builds.
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
+# speed-up the allocation.  However this would not be scalable to multiple
+# builds.
 .build_blueos_3_ppc64le_ib_script:
   script:
     - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test

.gitlab/lassen.yml

@@ -22,6 +22,7 @@
 
 # Spack helped builds
 # Generic lassen build job, extending build script
+# Note: Lassen jobs can start as soon as the setup job is complete.
 .build_and_test_on_lassen:
   extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
   stage: l_build_and_test

.gitlab/quartz.yml

@@ -16,13 +16,13 @@
     - shell
     - quartz
   rules:
-    # Don’t run quartz jobs if...
+    # Don't run quartz jobs if...
     - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
       when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
       when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status
@@ -37,6 +37,18 @@
     # Default is to run if previous stage succeeded
     - when: on_success
 
+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
 # Allocate
 q_allocate_resources:
   variables:
@@ -65,10 +77,11 @@ q_report_success:
   stage: q_release_resources
   script:
     - echo "Can only run if all the quartz jobs passed"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
     - git push origin master
@@ -80,10 +93,11 @@ q_report_failure:
   stage: q_release_resources
   script:
     - echo "Runs if there was at least one failure on quartz"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
@@ -94,7 +108,6 @@ q_report_failure:
 .build_and_test_on_quartz:
   extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
   stage: q_build_and_test
-  needs: [setup]
 
 # Build MFEM
 debug_ser_gcc_4_9_3:
@@ -137,7 +150,11 @@ opt_par_gcc_6_1_0_pumi:
     SPEC: "%gcc@6.1.0 +pumi"
   extends: .build_and_test_on_quartz
 
-# Baseline
+# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
+# that "setup-baseline" can start immediately. Then, we have to use needs for
+# each one of the baseline jobs, otherwise they will wait for the rest of the
+# pipeline.
+
 baselinecheck_mfem_intel_quartz:
   extends: [.baselinecheck_mfem, .on_quartz]
   needs: [setup_baseline]
@@ -147,14 +164,15 @@ update_autotest:
   needs: [baselinecheck_mfem_intel_quartz]
   stage: baseline_to_autotest
   script:
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
     # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
     - |
       if [[ -f ${rundir}/*.err ]]
       then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
         cp ${rundir}/*.err ${rundir}/autotest-email.html
       fi
     - git add ${rundir}

CHANGELOG

@@ -8,40 +8,71 @@
                                https://mfem.org
 
 
-Version 4.2.1 (development)
+Version 4.3.1 (development)
 ===========================
-- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
 
-- Memory management:
-  * Added method Device::SetMemoryTypes that can be used to change the default
-    host and device MemoryTypes before Device setup.
-  * In class MemoryManager, added methods GetDualMemoryType and
-    SetDualMemoryType; dual MemoryTypes are used to determine the second
-    MemoryType (host or device) when only one MemoryType is specified in methods
-    of class Memory.
-  * Added Memory constructor for setting both the host and device MemoryTypes.
-  * Switched the default behavior of device memory allocations so that they
-    are deferred until the device pointer is needed.
-  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
-    corresponding allocator that can be set with the method
-    MemoryManager::SetUmpireDevice2AllocatorName.
-  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
 
-- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
+Version 4.3, released on July 29, 2021
+======================================
 
-- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
+Discretization improvements
+---------------------------
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.
 
-- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Extended the support for field transfer between high-order and low-order
+  refined finite element spaces to include: dual fields and H1 fields (both
+  primary and dual). These are illustrated in the lor-transfer miniapp.
 
-- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of
-  the domain decomposition.
+- Improved libCEED integration, including support for VectorCoefficient,
+  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Changed the interface for the error estimator and implemented the Kelly error
+  indicator for scalar-valued problems, supported in serial and parallel builds.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+Linear and nonlinear solvers
+----------------------------
+- Added support for AMG preconditioners on GPUs based on the hypre library
+  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
+  MFEM examples that use hypre have been ported to support this functionality.
+  The GPU preconditioners require that both hypre and MFEM are built with CUDA
+  support. Hypre builds with CUDA and unified memory are also supported and
+  can be used with `-d cuda:uvm` as a command-line option.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+
+- Generalized the Multigrid class to support non-geometric multigrid. Previous
+  functionality, based on FiniteElementSpaceHierarchy, is now available in the
+  derived class GeometricMultigrid.
 
 - Introduced solver interface for linear problems with constraints, a few
   concrete solvers that implement the interface, and a demonstration of their
@@ -52,19 +83,18 @@ Version 4.2.1 (development)
   as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
   Example 3p and linalg/auxiliary.?pp.
 
-- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
+- Improved interface for using the Ginkgo library, including: support for matrix-
+  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
+  support, and reduction of unnecessary data copies.
 
-- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p, ex3p, and ex4p, and may not work in more general settings.
 
-- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.
 
 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
   See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -72,6 +102,17 @@ Version 4.2.1 (development)
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
   ESDIRK-32, and A-stable ESDIRK-33.
 
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+Meshing improvements
+--------------------
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
   of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
   parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -80,113 +121,26 @@ Version 4.2.1 (development)
   NC data files are compatible with serial code, e.g., can be viewed with serial
   GLVis. Loading of legacy NC mesh files is still supported.
 
-- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
-- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
-- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
-- Generalized the Multigrid class to support non-geometric multigrid. The
-  previous functionality, based on FiniteElementSpaceHierarchy, is now available
-  in the derived class GeometricMultigrid.
-
-- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
-     and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
-- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
-- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
-- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
-- Added HIP support to the CMake build system.
-
-- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
-- Added support for reading VTK meshes in XML format.
-
-- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
-- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
+  unstructured high-order meshes with general high-order finite element fields
+  on them. When enabled, mfem can convert data collections to/from FMS data
+  collections in memory. In addition, an FMS data collection class was added so
+  the convert-dc miniapp can read and generate data files in FMS format.
 
 - Added new mesh quality metrics and improved the untangling capabilities of the
   TMOP-based mesh optimization algorithms.
 
-- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
-- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
-- Changed the interface for the error estimator.
-
-- Implemented the Kelly error indicator for scalar-valued problems, supported
-  in serial and parallel builds.
-
-- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
-- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
-- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
-- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
+- The TMOP mesh optimization algorithms were extended to GPU:
+  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
+  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+  * Kernels for normalization and limiting have been added
+  * The AdvectorCG now also supports AssemblyLevel::PARTIAL
 
 - Added support for creating refined meshes for all element types (e.g. by
   splitting high-order elements into low-order refined elements), including
   mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
   supports meshes with any element geometry.
 
-- Testing improvements:
-  * Transitioned from Travis to GitHub Action for testing/CI on GitHub.
-  * Effectively remove Travis from CI.
-  * Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
-  * Added a set of suggested git hooks for developers in config/githooks.
-
-- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
-- Added a new, very simple example (ex0 and parallel version ex0p). This
-  example solves a simple Poisson problem using H1 elements (the same problem as
-  ex1), but is intended to be extremely simple and approachable for new users.
-
 - Meshes consisting of any type of elements (including mixed meshes) can be
   converted to all-simplex meshes using Mesh::MakeSimplicial.
 
@@ -199,42 +153,133 @@ Version 4.2.1 (development)
   requisite periodic vertex mappings can be created with
   Mesh::CreatePeriodicVertexMapping.
 
-- Added support for transferring dual fields between high-order and low-order
-  refined finite element spaces using the transposed versions of the
-  L2ProjectionGridTransfer operators. This functionality is illustrated in the
-  lor-transfer miniapp.
-
-- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
-- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p and may not work in more general settings.
-
-- Added support for transferring fields (primary and dual) between high-order
-  and low-order refined H1 finite element spaces using the
-  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
-  through the lor-transfer miniapp when run with the -h1 option.
-
-- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).
 
 - Added sample meshes in the `data` subdirectory showing the reference elements
   of the six currently supported element types; ref-segment.mesh,
   ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
   ref-prism.mesh.
 
+High-performance computing
+--------------------------
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Added method Device::SetMemoryTypes that can be used to change the default
+  host and device MemoryTypes before Device setup.
+
+- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
+  dual MemoryTypes are used to determine the second MemoryType (host or device)
+  when only one MemoryType is specified in methods of class Memory.
+
+- Added Memory constructor for setting both the host and device MemoryTypes.
+
+- Switched the default behavior of device memory allocations so that they are
+  deferred until the device pointer is needed.
+
+- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
+  allocator that can be set with the method SetUmpireDevice2AllocatorName.
+
+- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
+
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new, very simple example (ex0 and parallel version ex0p). This example
+  solves a simple Poisson problem using H1 elements (the same problem as ex1),
+  but is intended to be extremely simple and approachable for new users.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.
+
 - Added a high-order extension of the shifted boundary method to solve PDEs on
   non body-fitted meshes. This is illustrated in the new Shifted Diffusion
   miniapp, see miniapps/shifted/diffusion.cpp.
 
+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.
+
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of the
+  domain decomposition.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+Improved testing
+----------------
+- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
+
+- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
+
+- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
+  support.
+
+- Added a set of suggested git hooks for developers in config/githooks.
+
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+Miscellaneous
+-------------
+- The following integrations have updated minimum version requirements:
+  * CUDA >= 10.1.168
+  * Ginkgo >= 1.4.0
+  * GSLIB >= 1.0.7
+  * HIOP >= 0.4
+  * HYPRE >= 2.20.0 for mixedint support
+  * HYPRE >= 2.22.0 for CUDA support
+  * libCEED >= 0.8
+  * PETSc >= 3.15.0 for CUDA support
+  * RAJA >= 0.13.0
+  see INSTALL for more details.
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for reading VTK meshes in XML format.
+
 - Added makefile rule to generate TAGS table for vi or Emacs users.
 
+- Added HIP support to the CMake build system.
+
+- Various other simplifications, extensions, and bugfixes in the code.
+
 API changes
 -----------
 - Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
@@ -244,20 +289,11 @@ API changes
   `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
   `mfem::FaceRestriction::MultTranspose`.
 
-libCEED integration improvements
---------------------------------
-- Refactor the libCEED integration
-
-- Add support for VectorCoefficient with libCEED backends.
-
-- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
-  libCEED backends.
-
 
 Version 4.2, released on October 30, 2020
 =========================================
 
-High-Performance Computing
+High-performance computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
   code, which can now take advantage of specific classes on the following
@@ -339,9 +375,6 @@ Linear and nonlinear solvers
   matrix with the function HypreParMatrixFromBlocks. This could be useful for
   solving block systems with parallel direct solvers such as STRUMPACK.
 
-- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
   preconditioner. The latter requires hypre version 2.19.0 or later.
 
@@ -452,7 +485,7 @@ New and updated examples and miniapps
     L2, with partial assembly support in Example 24/24p.
   * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
 
-Data management and Visualization
+Data management and visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
   Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection

CMakeLists.txt

@@ -16,7 +16,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO) 
+if (MFEM_USE_GINKGO)
   set(CMAKE_CXX_STANDARD 14)
 endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -54,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.2.1)
+set(${PROJECT_NAME}_VERSION 4.3.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,7 +102,7 @@ if (MFEM_USE_CUDA)
    endif()
    enable_language(CUDA)
    set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO) 
+   if (MFEM_USE_GINKGO)
      set(CMAKE_CUDA_STANDARD 14)
    endif()
    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
@@ -246,6 +246,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
     message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
   endif()
   find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()
 
 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -266,6 +267,15 @@ if (MFEM_USE_SUNDIALS)
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
+# EPIC
+if (MFEM_USE_EPIC)
+  if (NOT (MFEM_USE_MPI AND MFEM_USE_SUNDIALS AND MFEM_USE_LAPACK) )
+    message(FATAL_ERROR " *** EPIC requires that MPI, SUNDIALS and LAPACK be enabled.")
+  else()
+    find_package(EPIC REQUIRED SUNDIALS NVector_Serial NVector_Parallel BLAS LAPACK)
+  endif()
+endif()
+
 # Mesquite
 if (MFEM_USE_MESQUITE)
   find_package(Mesquite REQUIRED)
@@ -330,6 +340,10 @@ if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
 
+if (MFEM_USE_FMS)
+    find_package(FMS REQUIRED fms )
+endif()
+
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Axom)
@@ -423,10 +437,11 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS EPIC PETSC
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
     CUSPARSE MKL_CPARDISO AMGX CALIPER)
+
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -445,6 +460,9 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
   message(STATUS "MFEM: using package OpenMP")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
 endif()
 
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")

CONTRIBUTING.md

@@ -97,7 +97,6 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   ├── cmake
-  │   │    └── ...
   │   └── githooks
   ├── data
   ├── doc
@@ -135,10 +134,10 @@ The MFEM source code has the following structure:
   └── tests
       ├── convergence
       ├── gitlab
+      ├── mem_manager
       ├── par-mesh-format
       ├── scripts
       └── unit
-          └── ...
 ```
 
 #### Main directories and classes

INSTALL

@@ -474,7 +474,7 @@ MFEM_USE_HIP = YES/NO
    Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
    interface for portability developed by AMD that can target both AMD and
    NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
    option uses the HIP_* build options, see below.
 
 MFEM_USE_RAJA = YES/NO
@@ -516,6 +516,13 @@ MFEM_USE_CALIPER = YES/NO
    profiling at runtime with Caliper's configuration API. Alternatively, one
    can configure Caliper through environment variables or config files.
 
+MFEM_USE_FMS = YES/NO
+   Enables support for the FMS library which consists of the DataCollection
+   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
+   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   DataCollection class, see the header file fem/fmsconvert.hpp.
+
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -540,8 +547,9 @@ The specific libraries and their options are:
   See also the "Specific options for hypre" section at the end of this file.
   URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b,
-            HYPRE >= 2.20.0 for '--enable-mixedint' support.
+  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
+            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
+            HYPRE >= 2.22.0   (HYPRE built with CUDA)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -615,7 +623,7 @@ The specific libraries and their options are:
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0. 
+  Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX
@@ -753,6 +761,11 @@ The specific libraries and their options are:
   URL: https://zlib.net
   Options: ZLIB_OPT, ZLIB_LIB.
 
+- FMS (optional), used when MFEM_USE_FMS = YES.
+  URL: https://github.com/CEED/FMS
+  Options: FMS_OPT, FMS_LIB.
+  Versions: FMS >= 0.2.
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -884,6 +897,7 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
+MFEM_USE_FMS
 
 The following options are CMake specific:
 
@@ -938,6 +952,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - UMPIRE
  - AXOM - Used when MFEM_USE_SIDRE is enabled
  - CALIPER
+ - FMS
 
 The following built-in CMake packages are also used:
 
@@ -955,7 +970,7 @@ config/config.hpp.in:
 
    cp config/config.hpp.in config/_config.hpp
 
-The file config/_config.hpp can then be edited to enable desired options.  The
+The file config/_config.hpp can then be edited to enable desired options. The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.
 
@@ -963,7 +978,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.
 
 MFEM will expect the launcher command, plus the command line option to allow it

config/XSDKDefaults.cmake

@@ -256,6 +256,10 @@ IF (DEFINED TPL_ENABLE_SIDRE)
   SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_FMS)
+  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_CONDUIT)
   SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -29,6 +29,7 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_EPIC @MFEM_USE_EPIC@)
 set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
@@ -44,6 +45,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
+set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)

config/cmake/config.hpp.in

@@ -119,6 +119,9 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD
 
+// Enable MFEM functionality based on the FMS library
+#cmakedefine MFEM_USE_FMS
+
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 
@@ -162,6 +165,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 #cmakedefine MFEM_USE_SUNDIALS
 
+// Enable MFEM functionality based on the EPIC libraries.
+#cmakedefine MFEM_USE_EPIC
+
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 

config/cmake/modules/FindEPIC.cmake

@@ -0,0 +1,21 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - EPIC_FOUND
+#   - EPIC_LIBRARIES
+#   - EPIC_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(EPIC EPIC EPIC_DIR
+       "include" Epic.h "lib" epic1.0.0
+  "Paths to headers required by EPIC." "Libraries required by EPIC.")
+

config/cmake/modules/FindFMS.cmake

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - FMS_FOUND
+#   - FMS_LIBRARIES
+#   - FMS_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(FMS FMS FMS_DIR
+  "include" fms.h "lib" fms
+  "Paths to headers required by FMS." "Libraries required by FMS.")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -759,7 +759,7 @@ function(mfem_export_mk_files)
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_EPIC MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
       MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
       MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI

config/config.hpp.in

@@ -85,6 +85,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS
 
+// Enable MFEM functionality based on the EPIC libraries.
+// #define MFEM_USE_EPIC
+
 // Enable MFEM functionality based on the Mesquite library.
 // #define MFEM_USE_MESQUITE
 
@@ -117,6 +120,9 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD
 
+// Enable FMS support
+// #define MFEM_USE_FMS
+
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT
 

config/config.mk.in

@@ -29,6 +29,7 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_EPIC          = @MFEM_USE_EPIC@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
@@ -43,6 +44,7 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
+MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@

config/defaults.cmake

@@ -30,6 +30,7 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_EPIC "Enable EPIC usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
@@ -45,6 +46,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
+option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -96,6 +98,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -109,6 +116,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")
 
+set(EPIC_DIR "${MFEM_DIR}/../epic-cpp/instdir" CACHE PATH
+    "Path to the EPIC library.")
+
 set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
     "Path to the Mesquite library.")
 
@@ -132,10 +142,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
     "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")    
+    "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")    
+#     "Additional libraries required by MUMPS.")
 
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
     "Path to the STRUMPACK library.")
@@ -187,6 +197,12 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
 
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")
 
+set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
+    "Path to the FMS library.")
+# If FMS is built with Conduit:
+# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
+#     "Additional packages required by FMS.")
+
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
     "Path to the Conduit library.")
 

config/defaults.mk

@@ -122,6 +122,7 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_EPIC          = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
@@ -136,6 +137,7 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
+MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -174,6 +176,10 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
+ifeq (YES,$(MFEM_USE_CUDA))
+   # This is only necessary when hypre is built with cuda:
+   HYPRE_LIB += -lcusparse -lcurand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -226,6 +232,11 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
+# EPIC library configuration
+MESQUITE_DIR = @MFEM_DIR@/../epic-cpp/instdir
+MESQUITE_OPT = -I$(EPIC_DIR)/include
+MESQUITE_LIB = -L$(EPIC_DIR)/lib -lepic1.0.0
+
 # MESQUITE library configuration
 MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
 MESQUITE_OPT = -I$(MESQUITE_DIR)/include
@@ -357,6 +368,11 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr
 
+# FMS and required libraries configuration
+FMS_DIR = $(MFEM_DIR)/../fms
+FMS_OPT = -I$(FMS_DIR)/include
+FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
+
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit

config/test.mk

@@ -57,22 +57,27 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code
+# Test runs of the examples/miniapps with parameters - check exit code:
+# 0 means success, 255 means the test was skipped, anything else means error
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
-   if [ "$$3" = 0 ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
-   rm -f $(1).stderr; exit $$3
+   err="$$3"; \
+   if [ "$$3" = 0 ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
+   rm -f $(1).stderr; exit $$err
 
 # Test runs of the examples/miniapps - check exit code and if a file exists
+# See mfem-test for the interpretation of the error code
 mfem-test-file = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
    rm -f $(1).stderr; exit $$err
 
 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -80,6 +85,26 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
+ifeq ($(MFEM_USE_CUDA),YES)
+.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
+test-par-YES: test-par-YES-cuda
+test-par-NO:  test-par-NO-cuda
+test-par-YES-cuda: test-par-cuda test-ser-cuda
+test-par-NO-cuda:  test-ser-cuda
+test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
+test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
+test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
+endif
+ifeq ($(MFEM_USE_HIP),YES)
+.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
+test-par-YES: test-par-YES-hip
+test-par-NO:  test-par-NO-hip
+test-par-YES-hip: test-par-hip test-ser-hip
+test-par-NO-hip:  test-ser-hip
+test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
+test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
+test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
+endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec

data/star-q3.fms

@@ -0,0 +1,246 @@
+FMS: 100
+DataCollection/Name: star
+DataCollection/NumberOfFieldDescriptors: 1
+DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
+DataCollection/FieldDescriptors/0/ComponentName: volume
+DataCollection/FieldDescriptors/0/Type: 0
+DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
+DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
+DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
+DataCollection/FieldDescriptors/0/NumDofs: 211
+DataCollection/NumberOfFields: 1
+DataCollection/Fields/0/Name: Coords
+DataCollection/Fields/0/LayoutType: 0
+DataCollection/Fields/0/NumberOfVectorComponents: 2
+DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
+DataCollection/Fields/0/Data/Size: 422
+DataCollection/Fields/0/Data/Type: FMS_DOUBLE
+DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
+1.309020, -0.809017, -0.500000, 
+-0.809017, -1.618030, 0.309017, 
+-0.500000, 1.309020, 0.519420, 
+1.154510, 0.809019, 0.147680, 
+-0.095492, -0.654508, -0.415586, 
+-1.213520, -1.213520, -0.392210, 
+-0.654508, -0.095492, 0.139949, 
+0.809019, 1.154510, 0.660184, 
+-0.264063, -0.800064, -0.231060, 
+0.663691, 0.183114, 0.317639, 
+0.543082, 0.598483, 0.345112, 
+0.478298, 0.027703, 0.095229, 
+0.012368, -0.092534, -0.334412, 
+-0.313767, -0.140526, -0.293881, 
+-0.534056, -0.660290, -0.537646, 
+-0.655590, -0.121396, -0.274504, 
+-0.346497, -0.296570, 0.004737, 
+-0.098835, 0.069287, 0.082675, 
+0.318799, 0.467183, 0.564505, 
+0.595190, 0.846237, 0.671735, 
+1.051500, 1.103010, 0.964008, 
+0.821603, 1.257520, 1.206010, 
+1.142350, 0.975686, 0.781273, 
+0.717257, 0.475684, 0.642352, 
+0.268930, 0.211049, 0.174181, 
+0.039345, -0.147746, -0.177481, 
+-0.365164, -0.230328, -0.551503, 
+-0.603005, -0.497587, -0.389864, 
+-0.757514, -0.706011, -0.675487, 
+-0.528946, -0.943851, -1.078690, 
+-1.087600, -0.955467, -1.483190, 
+-1.348360, -1.483190, -1.348360, 
+-1.085930, -0.938010, -0.943851, 
+-1.078690, -0.681476, -0.540944, 
+-0.757514, -0.706011, -0.540614, 
+-0.367058, -0.551503, -0.603005, 
+-0.365164, -0.230328, -0.138552, 
+-0.206896, 0.174181, 0.039345, 
+0.268468, 0.222269, 0.475684, 
+0.642352, 0.759791, 0.719381, 
+1.142350, 0.975686, 1.257520, 
+1.206010, 0.972837, 0.836119, 
+1.051500, 1.103010, 0.214572, 
+0.407449, 0.288323, 0.449827, 
+-0.086700, -0.027358, -0.200560, 
+-0.166595, -0.271802, -0.418426, 
+-0.426131, -0.551441, -0.096117, 
+-0.206969, -0.027946, -0.184969, 
+0.211136, 0.260131, 0.407172, 
+0.430781, 0.718277, 0.885068, 
+0.753103, 0.957692, 0.866273, 
+1.024530, 0.934099, 1.093820, 
+0.348422, 0.524463, 0.404903, 
+0.587376, 0.054525, 0.146431, 
+-0.078026, -0.007795, -0.329488, 
+-0.302967, -0.488115, -0.439332, 
+-0.498515, -0.453527, -0.633059, 
+-0.570251, -0.655787, -0.791132, 
+-0.802013, -0.956872, -1.094720, 
+-1.208560, -1.207720, -1.339910, 
+-0.693371, -0.795404, -0.803594, 
+-0.945068, -0.464668, -0.631721, 
+-0.458968, -0.547876, -0.332878, 
+-0.485696, -0.286385, -0.408481, 
+0.053021, -0.058200, 0.110846, 
+-0.002086, 0.381384, 0.416784, 
+0.551133, 0.613261, 0.872474, 
+0.901208, 1.038300, 1.084660, 
+0.737459, 0.751250, 0.890002, 
+0.915210, 0.010915, 0.000000, 
+0.951057, 0.951057, 0.587785, 
+1.538840, -0.587785, 0.000000, 
+-0.951057, -1.538840, -0.951057, 
+-0.015847, 0.475529, 0.951057, 
+0.492248, 1.244950, 1.063310, 
+0.274399, 0.293893, -0.293892, 
+-0.296404, -1.063310, -1.244950, 
+-0.453865, -0.951057, -0.475529, 
+0.466620, 0.792932, -0.013913, 
+-0.748783, -0.497528, 0.021382, 
+-0.017158, 0.172591, 0.330125, 
+0.458568, 0.457971, 0.137740, 
+0.299049, 0.588394, 0.667324, 
+0.432341, 0.634346, 0.117322, 
+0.193603, 0.211702, 0.098278, 
+-0.199438, -0.077304, -0.082243, 
+-0.216296, -0.458634, -0.592374, 
+-0.563926, -0.680404, -0.135751, 
+-0.302942, -0.469005, -0.453640, 
+-0.182727, -0.314240, 0.024270, 
+0.021546, 0.158510, 0.317019, 
+0.485799, 0.492951, 0.792548, 
+0.634038, 0.951057, 0.951057, 
+0.777915, 0.613430, 0.951057, 
+0.951057, 0.793994, 0.635800, 
+1.049020, 1.146990, 1.084480, 
+0.924310, 1.440880, 1.342910, 
+1.380330, 1.221820, 0.948209, 
+0.856297, 0.746293, 0.904802, 
+0.476242, 0.393234, 0.489821, 
+0.391857, 0.194471, 0.075751, 
+0.097964, 0.195929, -0.097964, 
+-0.195928, -0.173234, -0.078922, 
+-0.489821, -0.391856, -0.467007, 
+-0.397859, -0.746293, -0.904802, 
+-0.945206, -0.849559, -1.380330, 
+-1.221820, -1.440880, -1.342910, 
+-1.100830, -0.923191, -1.049020, 
+-1.146990, -0.774515, -0.621542, 
+-0.951057, -0.951057, -0.803055, 
+-0.635255, -0.951057, -0.951057, 
+-0.792548, -0.634038, -0.454301, 
+-0.479369, -0.158510, -0.317019, 
+0.149331, 0.178643, 0.295860, 
+0.306275, 0.246225, 0.406610, 
+0.367954, 0.498458, -0.014929, 
+0.084917, -0.094272, -0.000726, 
+-0.245374, -0.340755, -0.435351, 
+-0.490564, -0.176355, -0.294974, 
+-0.148010, -0.328246, 0.136248, 
+0.182883, 0.328957, 0.309903, 
+0.646446, 0.622546, 0.800859, 
+0.801803, 0.643087, 0.654473, 
+0.796963, 0.816799, 0.748189, 
+0.898148, 0.819092, 0.970033, 
+1.049540, 1.162660, 1.115310, 
+1.266700, 0.539960, 0.728396, 
+0.656318, 0.783152, 0.302716, 
+0.403858, 0.219006, 0.309252, 
+0.009667, 0.107161, -0.118211, 
+0.011786, -0.284704, -0.179858, 
+-0.411567, -0.296676, -0.570168, 
+-0.635348, -0.726784, -0.793211, 
+-1.016860, -1.112970, -1.170910, 
+-1.281530, -0.737860, -0.851723, 
+-0.883732, -0.995070, -0.653153, 
+-0.769939, -0.631918, -0.797688, 
+-0.616289, -0.806819, -0.638485, 
+-0.790356, -0.136399, -0.322769, 
+-0.165339, -0.309622]
+DataCollection/Mesh/PartitionInfo/Size: 2
+DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
+DataCollection/Mesh/PartitionInfo/Values: [0, 1]
+DataCollection/Mesh/NumDomainNames: 1
+DataCollection/Mesh/NumComponents: 1
+DataCollection/Mesh/NumTags: 0
+DataCollection/Mesh/DomainNames/0/Name: Domain
+DataCollection/Mesh/DomainNames/0/NumDomains: 1
+DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
+DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
+11, 26, 14, 
+14, 0, 27, 
+14, 27, 17, 
+17, 0, 28, 
+17, 28, 20, 
+20, 0, 29, 
+20, 29, 23, 
+23, 0, 30, 
+23, 30, 11, 
+11, 1, 12, 
+1, 26, 12, 
+12, 3, 13, 
+3, 26, 13, 
+13, 2, 14, 
+2, 15, 2, 
+27, 15, 15, 
+5, 16, 5, 
+27, 16, 16, 
+4, 17, 4, 
+18, 4, 28, 
+18, 18, 7, 
+19, 7, 28, 
+19, 19, 6, 
+20, 6, 21, 
+6, 29, 21, 
+21, 9, 22, 
+9, 29, 22, 
+22, 8, 23, 
+8, 24, 8, 
+30, 24, 24, 
+10, 25, 10, 
+30, 25, 25, 1]
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
+3, 3, 4, 
+5, 6, 6, 
+7, 8, 9, 
+9, 10, 11, 
+12, 12, 13, 
+14, 0, 15, 
+16, 17, 1, 
+17, 18, 19, 
+20, 2, 20, 
+21, 22, 22, 
+23, 24, 4, 
+24, 25, 26, 
+27, 5, 27, 
+28, 29, 29, 
+30, 31, 7, 
+31, 32, 33, 
+34, 8, 34, 
+35, 36, 36, 
+37, 38, 10, 
+38, 39, 40, 
+41, 11, 41, 
+42, 43, 43, 
+44, 45, 13, 
+45, 46, 47, 
+48, 14, 48, 
+49, 15]
+DataCollection/Mesh/Components/0/Name: volume
+DataCollection/Mesh/Components/0/Dimension: 2
+DataCollection/Mesh/Components/0/NumEntities: 20
+DataCollection/Mesh/Components/0/Coordinates: Coords
+DataCollection/Mesh/Components/0/NumParts: 1
+DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
+DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
+DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
+DataCollection/Mesh/Components/0/Relations/Size: 0
+DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.2.1
+PROJECT_NUMBER         = v4.3.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

examples/CMakeLists.txt

@@ -84,8 +84,9 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
-    set(THIS_TEST_OPTIONS "-no-vis")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  if (${TEST_NAME} MATCHES "ex0p?")
+    set(THIS_TEST_OPTIONS)
   endif()
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -107,6 +108,34 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   endif()
 endforeach()
 
+# Add CUDA/HIP tests.
+set(DEVICE_EXAMPLES
+    # serial examples with device support:
+    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+    # parallel examples with device support:
+    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+set(MFEM_TEST_DEVICE)
+if (MFEM_USE_CUDA)
+  set(MFEM_TEST_DEVICE "cuda")
+elseif (MFEM_USE_HIP)
+  set(MFEM_TEST_DEVICE "hip")
+endif()
+if (MFEM_TEST_DEVICE)
+  foreach(TEST_NAME ${DEVICE_EXAMPLES})
+    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    elseif (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
+
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
   add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
@@ -130,6 +159,11 @@ if (MFEM_USE_AMGX)
   add_subdirectory(amgx)
 endif()
 
+# Include the examples/epic directory if EPIC is enabled.
+if (MFEM_USE_EPIC)
+  add_subdirectory(epic)
+endif()
+
 # Include the examples/ginkgo directory if GINKGO is enabled.
 if (MFEM_USE_GINKGO)
   add_subdirectory(ginkgo)

examples/caliper/README

@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the Caliper performance profiling library.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
+2.5.0 of Caliper is recommended, though older versions may work too).
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.

examples/epic/CMakeLists.txt

@@ -0,0 +1,64 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(EPIC_EXAMPLES_SRCS)
+list(APPEND EPIC_EXAMPLES_SRCS
+  ex16.cpp
+  )
+
+if (MFEM_USE_MPI)
+list(APPEND EPIC_EXAMPLES_SRCS
+    ex16p.cpp
+    )
+endif()
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_epic" target, see below.
+add_custom_target(test_epic
+  ${CMAKE_CTEST_COMMAND} -R epic USES_TERMINAL)
+
+# Add one executable per cpp file, adding "epic_" as prefix. Sets
+# "test_epic" as a target that depends on the given examples.
+set(PFX epic_)
+add_mfem_examples(EPIC_EXAMPLES_SRCS ${PFX} "" test_epic)
+
+# Testing.
+# The EPIC tests can be run separately using the target "test_epic"
+# which builds the examples and runs:
+#   ctest -R epic
+
+# Example 16: use the default options
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${EPIC_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
+

examples/epic/README

@@ -0,0 +1,17 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               http://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the EPIC suite of time integration.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_EPIC = YES".
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.

examples/epic/ex16.cpp

@@ -0,0 +1,610 @@
+//                                MFEM Example 16
+//                               EPIC Modification
+//
+// Compile with: make ex16
+//
+// Sample runs:  ex16
+//               ex16 -m ../../data/inline-tri.mesh
+//               ex16 -m ../../data/disc-nurbs.mesh -tf 2
+//               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
+//               ex16 -m ../../data/fichera-q2.mesh
+//               ex16 -m ../../data/escher.mesh
+//               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
+//               ex16 -m ../../data/amr-quad.mesh -o 4 -r 0
+//               ex16 -m ../../data/amr-hex.mesh -o 2 -r 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class ImplicitSolveOperator;
+class JacobianOperator;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-K(u) u)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   FiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   BilinearForm *M;
+   mutable BilinearForm *K;
+   mutable BilinearForm *dK;
+   mutable BilinearForm *J_K;
+
+   SparseMatrix Mmat;
+   mutable SparseMatrix J_K_mat;
+
+   mutable CGSolver M_solver; // Krylov solver for inverting the mass matrix M
+   DSmoother M_prec;  // Preconditioner for the mass matrix M
+
+   CGSolver Jg_solver; // Krylov solver for inverting the Jacobian in the nonlinear solve
+   DSmoother Jg_prec;  // Preconditioner for the Jacobian Jg
+
+   NewtonSolver newton_solver;
+   mutable JacobianOperator *jac;
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+   mutable int nRhsMult, nSetJac, nJacMult, nImpSolve, nImpIter, nImpMult, nImpSet;
+
+public:
+   Vector u0;
+
+   ConductionOperator(FiniteElementSpace &f, double alpha, double kappa, const Vector &u);
+
+   void UpdateStats();
+   void PrintStats(ostream& out);
+
+   void ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2);
+
+   BilinearForm& GetKLambda(const Vector& u) const;
+   BilinearForm& GetdKLambda(const Vector& u) const;
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+   virtual Operator& GetGradient(const Vector &k) const;
+
+   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
+
+   virtual ~ConductionOperator();
+};
+
+class ImplicitSolveOperator : public Operator
+{
+private:
+   double dt;
+   const Vector* x;
+   ConductionOperator* oper;
+
+   const SparseMatrix* M;
+   mutable SparseMatrix* Jg;
+
+   mutable Vector u, z;
+   mutable int nMult, nSet;
+
+public:
+   ImplicitSolveOperator(ConductionOperator* oper, const SparseMatrix* M, double dt, const Vector* x);
+
+   int GetnMult() { return nMult; }
+   int GetnSet() { return nSet; }
+   virtual void Mult(const Vector &k, Vector &gk) const;
+   virtual Operator &GetGradient(const Vector &k) const;
+};
+
+class JacobianOperator : public Operator
+{
+private:
+   Operator* J;
+   Operator* M_solver;
+
+   mutable int nMult;
+   mutable Vector z;
+public:
+   JacobianOperator(Operator* J, Operator* M_solver);
+
+   int GetnMult() { return nMult; }
+
+   void ExtractJacobian(const Vector& x, std::ostream &out);
+   virtual void Mult(const Vector &k, Vector &gk) const;
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ref_levels = 2;
+   int order = 2;
+   int ode_solver_type = 8; // Exponential Euler
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver:\n\t"
+                  "1  - Forward Euler,\n\t"
+                  "2  - RK2,\n\t"
+                  "3  - RK3 SSP,\n\t"
+                  "4  - RK4,\n\t"
+                  "5  - Backward Euler,\n\t"
+                  "6  - SDIRK 2,\n\t"
+                  "7  - SDIRK 3,\n\t"
+                  "8  - EPIC (exponential euler)\n\t");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (ode_solver_type < 1 || ode_solver_type > 9)
+   {
+      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      return 3;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 3. Refine the mesh to increase the resolution. In this example we do
+   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
+   //    command-line parameter.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   FiniteElementSpace fespace(mesh, &fe_coll);
+
+   int fe_size = fespace.GetTrueVSize();
+   cout << "Number of temperature unknowns: " << fe_size << endl;
+
+   GridFunction u_gf(&fespace);
+
+   // 5. Set the initial conditions for u. All boundaries are considered
+   //    natural.
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 6. Initialize the conduction operator and the visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ofstream omesh("ex16.mesh");
+      omesh.precision(precision);
+      mesh->Print(omesh);
+      ofstream osol("ex16-init.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16", mesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         cout << "Unable to connect to GLVis server at "
+              << vishost << ':' << visport << endl;
+         visualization = false;
+         cout << "GLVis visualization disabled.\n";
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *mesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
+   }
+
+   // 7. Define the ODE solver used for time integration.
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // MFEM explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
+      // EPIC 
+      case 8: ode_solver = new EPI2();break;
+      case 9: ode_solver = new EPIRK4(); break;
+   }
+
+   // Initialize integrators
+   ode_solver->Init(oper);
+
+   // 8. Perform time-integration (looping over the time iterations, ti, with a
+   //    time-step dt).
+   cout << "Integrating the ODE ..." << endl;
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   /*ofstream out_jac_an("jacobian_an.txt");
+   ofstream out_jac_fd("jacobian_fd.txt");
+   oper.ExtractJacobians(u, out_jac_fd, out_jac_an);*/
+
+   bool last_step = false;
+   int ti;
+   for (ti = 1; !last_step; ti++)
+   {
+      double dt_real = min(dt, t_final - t);
+
+      // Note that since we are using the "one-step" mode of the SUNDIALS
+      // solvers, they will, generally, step over the final time and will not
+      // explicitly perform the interpolation to t_final as they do in the
+      // "normal" step mode.
+      ode_solver->Step(u, t, dt_real);
+
+      oper.UpdateStats();
+
+      last_step = (t >= t_final - 1e-8*dt);
+
+      if (last_step || (ti % vis_steps) == 0) {
+         cout << "step " << ti << ", t = " << t << endl;
+
+         u_gf.SetFromTrueDofs(u);
+         if (visualization) {
+            sout << "solution\n" << *mesh << u_gf << flush;
+         }
+
+         if (visit) {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+   }
+   tic_toc.Stop();
+   double comp_time = tic_toc.RealTime();
+   cout << "Done, " << comp_time << "s." << endl;
+
+   // 9. Save the final solution. This output can be viewed later using GLVis:
+   //    "glvis -m ex16.mesh -g ex16-final.gf".
+   {
+      ofstream osol("ex16-final.gf");
+      osol.precision(precision);
+      u_gf.Save(osol);
+
+      ofstream ostats("ex16-stats.txt");
+      ostats << "time " << comp_time << endl;
+      oper.PrintStats(ostats);
+   }
+
+   // 10. Free the used memory.
+   delete ode_solver;
+   delete mesh;
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al, double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL), dK(NULL), J_K(NULL), jac(NULL), z(height), u0(height),
+     nRhsMult(0), nSetJac(0), nJacMult(0), nImpSolve(0), nImpIter(0), nImpMult(0), nImpSet(0)
+{
+   const double rel_tol = 1e-8;
+
+   M = new BilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble();
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(50);
+   M_solver.SetPrintLevel(0);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(Mmat);
+
+   Jg_solver.SetRelTol(rel_tol);
+   Jg_solver.SetAbsTol(0.0);
+   Jg_solver.SetMaxIter(50);
+   Jg_solver.SetPrintLevel(0);
+   Jg_solver.SetPreconditioner(Jg_prec);
+
+   newton_solver.SetMaxIter(10);
+   newton_solver.SetRelTol(rel_tol);
+   newton_solver.SetPrintLevel(-1);
+   newton_solver.SetSolver(Jg_solver);
+   newton_solver.SetMaxIter(100);
+   newton_solver.iterative_mode = false;
+
+   alpha = al;
+   kappa = kap;
+}
+
+void ConductionOperator::UpdateStats()
+{
+   if (jac)
+   {
+      nJacMult += jac->GetnMult();
+   }
+}
+
+void ConductionOperator::PrintStats(ostream &out)
+{
+   out  << "nRhsMult " << nRhsMult << endl
+        << "nSetJac " << nSetJac << endl
+        << "nJacMult " << nJacMult  << endl
+        << "nImplicitSolve " << nImpSolve  << endl
+        << "nImplicitIter " << nImpIter << endl
+        << "nImplicitMult " << nImpMult << endl
+        << "nImplicitSet " << nImpSet << endl;
+}
+
+BilinearForm& ConductionOperator::GetKLambda(const Vector &u) const
+{
+   GridFunction conductivity_gf(&fespace);
+   conductivity_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < conductivity_gf.Size(); i++)
+   {
+      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
+   }
+
+   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
+
+   delete K;
+   K = new BilinearForm(&fespace);
+   K->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
+   K->Assemble();
+
+   return *K;
+}
+
+BilinearForm& ConductionOperator::GetdKLambda(const Vector &u) const
+{
+   GridFunction conductivity_gf(&fespace);
+   conductivity_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < conductivity_gf.Size(); i++)
+   {
+      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
+   }
+
+   // Define diffusion form with conductivity = kappa(u0)
+   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
+
+   // Define advection form with velocity = grad kappa(u0)
+   GridFunction neg_cond_gf(conductivity_gf);
+   neg_cond_gf.Neg();
+   GradientGridFunctionCoefficient velocity_coeff(&neg_cond_gf);
+
+   delete dK;
+   dK = new BilinearForm(&fespace);
+
+   dK->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
+   dK->AddDomainIntegrator(new MixedScalarWeakDivergenceIntegrator(velocity_coeff));
+   dK->Assemble();
+
+   return *dK;
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   GetKLambda(u);
+   K->Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+   nRhsMult++;
+}
+
+void ConductionOperator::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+{
+   ImplicitSolveOperator imp_oper(this, &this->Mmat, dt, &x);
+   newton_solver.SetOperator(imp_oper);
+
+   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
+   newton_solver.Mult(zero, k);
+   MFEM_VERIFY(newton_solver.GetConverged(), "Newton solver did not converge.");
+
+   nImpSolve++;
+   nImpMult += imp_oper.GetnMult();
+   nImpSet  += imp_oper.GetnSet();
+   nImpIter += newton_solver.GetNumIterations();
+}
+
+Operator &ConductionOperator::GetGradient(const Vector &u) const
+{
+   delete jac;
+   GetdKLambda(u);
+   jac = new JacobianOperator(dK, &M_solver);
+
+   nSetJac++;
+
+   return *jac;
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete M;
+   delete K;
+   delete dK;
+   delete J_K;
+   delete jac;
+}
+
+ImplicitSolveOperator::ImplicitSolveOperator(ConductionOperator *oper_, const SparseMatrix* M_, double dt_, const Vector* x_):
+   Operator(oper_->Height()), oper(oper_), M(M_), dt(dt_), x(x_), u(height), z(height), Jg(NULL), nMult(0), nSet(0)
+{ }
+
+
+void ImplicitSolveOperator::Mult(const Vector& y, Vector& gy) const
+{
+   // Compute gy = g(y) = My + dt K(lambda(u)) u
+   // with u = x + dt y
+   add(*x, dt, y, u);
+   BilinearForm& K = oper->GetKLambda(u);
+   K.Mult(u, gy);
+
+   M->AddMult(y, gy);
+
+   nMult++;
+}
+
+Operator& ImplicitSolveOperator::GetGradient(const Vector &k) const
+{
+   add(*x, dt, k, u);
+
+   BilinearForm& dK = oper->GetdKLambda(u);
+   Array<int> ess_tdof_list;
+   SparseMatrix dK_mat;
+   dK.FormSystemMatrix(ess_tdof_list, dK_mat);
+
+   delete Jg;
+   Jg = Add(1.0, *M, dt, dK_mat);
+
+   nSet++;
+   return *Jg;
+}
+
+JacobianOperator::JacobianOperator(Operator* J_, Operator* M_solver_):
+   Operator(M_solver_->Height()), J(J_), M_solver(M_solver_), z(height), nMult(0)
+{ }
+
+void JacobianOperator::Mult(const Vector &v, Vector &Jv) const
+{
+   Vector temp(v);
+   J->Mult(v, z);
+   z.Neg(); // z = -z
+   M_solver->Mult(z, Jv);
+   nMult++;
+}
+
+
+void ConductionOperator::ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2)
+{
+   int n = x.Size();
+
+   Vector e(n);
+   e = 0.0;
+
+   double eps = 1e-8;
+   Vector fx(n), fx_eps(n), x_eps(n);
+   Mult(x, fx);
+
+   DenseMatrix J(n);
+
+   for (int i = 0; i < n; i++)
+   {
+      e[i] = 1.0;
+      add(x, eps, e, x_eps);
+      Mult(x_eps, fx_eps);
+      fx_eps -= fx;
+      fx_eps /= eps;
+      J.SetCol(i, fx_eps);
+      e[i] = 0.0;
+   }
+
+   J.PrintMatlab(out);
+   GetGradient(x);
+   jac->ExtractJacobian(x, out2);
+}
+
+void JacobianOperator::ExtractJacobian(const Vector& x, std::ostream &out)
+{
+   int n = z.Size();
+
+   Vector e(n);
+   e= 0.0;
+
+   Vector J_i(n);
+   DenseMatrix J(n);
+
+   for (int i = 0; i < n; i++)
+   {
+      e[i] = 1.0;
+      Mult(e, J_i);
+      J.SetCol(i, J_i);
+      e[i] = 0.0;
+   }
+
+   J.PrintMatlab(out);
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5) { return 2.0; }
+   else                  { return 1.0; }
+}

examples/epic/ex16p.cpp

@@ -0,0 +1,494 @@
+//                         MFEM Example 16 - Parallel Version
+//                             SUNDIALS Modification
+//
+// Compile with: make ex16p
+//
+// Sample runs:
+//     mpirun -np 4 ex16p
+//     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
+//     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
+//     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
+//     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
+//     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
+//     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
+//     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
+//     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
+//     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
+//     mpirun -np 4 ex16p -m ../../data/amr-quad.mesh -o 4 -rs 0 -rp 0
+//     mpirun -np 4 ex16p -m ../../data/amr-hex.mesh -o 2 -rs 0 -rp 0
+//
+// Description:  This example solves a time dependent nonlinear heat equation
+//               problem of the form du/dt = C(u), with a non-linear diffusion
+//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
+//
+//               We recommend viewing examples 2, 9 and 10 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/** After spatial discretization, the conduction model can be written as:
+ *
+ *     du/dt = M^{-1}(-Ku)
+ *
+ *  where u is the vector representing the temperature, M is the mass matrix,
+ *  and K is the diffusion operator with diffusivity depending on u:
+ *  (\kappa + \alpha u).
+ *
+ *  Class ConductionOperator represents the right-hand side of the above ODE.
+ */
+class ConductionOperator : public TimeDependentOperator
+{
+protected:
+   ParFiniteElementSpace &fespace;
+   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
+
+   ParBilinearForm *M;
+   ParBilinearForm *K;
+
+   HypreParMatrix Mmat;
+   HypreParMatrix Kmat;
+   HypreParMatrix *T; // T = M + dt K
+   double current_dt;
+
+   CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
+   HypreSmoother M_prec; // Preconditioner for the mass matrix M
+
+   CGSolver T_solver;    // Implicit solver for T = M + dt K
+   HypreSmoother T_prec; // Preconditioner for the implicit solver
+
+   double alpha, kappa;
+
+   mutable Vector z; // auxiliary vector
+
+public:
+   ConductionOperator(ParFiniteElementSpace &f, double alpha, double kappa,
+                      const Vector &u);
+
+   virtual void Mult(const Vector &u, Vector &du_dt) const;
+
+   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
+       This is the only requirement for high-order SDIRK implicit integration.*/
+   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
+
+   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
+                                int jok, int *jcur, double gamma);
+
+   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
+       SUNDIALS solvers. */
+   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+
+   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
+   void SetParameters(const Vector &u);
+
+   virtual ~ConductionOperator();
+};
+
+double InitialTemperature(const Vector &x);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 2;
+   int par_ref_levels = 1;
+   int order = 2;
+   int ode_solver_type = 8; // Exponential Euler 
+   double t_final = 0.5;
+   double dt = 1.0e-2;
+   double alpha = 1.0e-2;
+   double kappa = 0.5;
+   bool visualization = true;
+   bool visit = false;
+   int vis_steps = 5;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver:\n\t"
+                  "1  - Forward Euler,\n\t"
+                  "2  - RK2,\n\t"
+                  "3  - RK3 SSP,\n\t"
+                  "4  - RK4,\n\t"
+                  "5  - Backward Euler,\n\t"
+                  "6  - SDIRK 2,\n\t"
+                  "7  - SDIRK 3,\n\t"
+                  "8  - Exponential Euler,\n\t");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&alpha, "-a", "--alpha",
+                  "Alpha coefficient.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "Kappa coefficient offset.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
+                  "--no-visit-datafiles",
+                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      MPI_Finalize();
+      return 1;
+   }
+
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // check for vaild ODE solver option
+   if (ode_solver_type < 1 || ode_solver_type > 8)
+   {
+      if (myid == 0)
+      {
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+      }
+      MPI_Finalize();
+      return 1;
+   }
+
+   // 3. Read the serial mesh from the given mesh file on all processors. We can
+   //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
+   //    with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the mesh in serial to increase the resolution. In this example
+   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
+   //    a command-line parameter.
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define the vector finite element space representing the current and the
+   //    initial temperature, u_ref.
+   H1_FECollection fe_coll(order, dim);
+   ParFiniteElementSpace fespace(pmesh, &fe_coll);
+
+   int fe_size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of temperature unknowns: " << fe_size << endl;
+   }
+
+   ParGridFunction u_gf(&fespace);
+
+   // 7. Set the initial conditions for u. All boundaries are considered
+   //    natural.
+   FunctionCoefficient u_0(InitialTemperature);
+   u_gf.ProjectCoefficient(u_0);
+   Vector u;
+   u_gf.GetTrueDofs(u);
+
+   // 8. Initialize the conduction operator and the VisIt visualization.
+   ConductionOperator oper(fespace, alpha, kappa, u);
+
+   u_gf.SetFromTrueDofs(u);
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "ex16-mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "ex16-init." << setfill('0') << setw(6) << myid;
+      ofstream omesh(mesh_name.str().c_str());
+      omesh.precision(precision);
+      pmesh->Print(omesh);
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   VisItDataCollection visit_dc("Example16-Parallel", pmesh);
+   visit_dc.RegisterField("temperature", &u_gf);
+   if (visit)
+   {
+      visit_dc.SetCycle(0);
+      visit_dc.SetTime(0.0);
+      visit_dc.Save();
+   }
+
+   socketstream sout;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      sout.open(vishost, visport);
+      sout << "parallel " << num_procs << " " << myid << endl;
+      int good = sout.good(), all_good;
+      MPI_Allreduce(&good, &all_good, 1, MPI_INT, MPI_MIN, pmesh->GetComm());
+      if (!all_good)
+      {
+         sout.close();
+         visualization = false;
+         if (myid == 0)
+         {
+            cout << "Unable to connect to GLVis server at "
+                 << vishost << ':' << visport << endl;
+            cout << "GLVis visualization disabled.\n";
+         }
+      }
+      else
+      {
+         sout.precision(precision);
+         sout << "solution\n" << *pmesh << u_gf;
+         sout << "pause\n";
+         sout << flush;
+         if (myid == 0)
+         {
+            cout << "GLVis visualization paused."
+                 << " Press space (in the GLVis window) to resume it.\n";
+         }
+      }
+   }
+
+   // 9. Define the ODE solver used for time integration.
+   double t = 0.0;
+   ODESolver *ode_solver = NULL;
+   EPICSolver *epic_solver = NULL;
+   switch (ode_solver_type)
+   {
+      // MFEM explicit methods
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 5: ode_solver = new BackwardEulerSolver; break;
+      case 6: ode_solver = new SDIRK23Solver(2); break;
+      case 7: ode_solver = new SDIRK33Solver; break;
+      // EPIC
+      case 8:
+         epic_solver = new EPICSolver();
+	 epic_solver->Init(oper);
+	 ode_solver = epic_solver;
+	 break;
+   }
+
+   // Initialize MFEM integrators
+   ode_solver->Init(oper);
+
+   // 10. Perform time-integration (looping over the time iterations, ti, with a
+   //     time-step dt).
+   if (myid == 0)
+   {
+      cout << "Integrating the ODE ..." << endl;
+   }
+   tic_toc.Clear();
+   tic_toc.Start();
+
+   bool last_step = false;
+   for (int ti = 1; !last_step; ti++)
+   {
+      double dt_real = min(dt, t_final - t);
+
+      // Note that since we are using the "one-step" mode of the SUNDIALS
+      // solvers, they will, generally, step over the final time and will not
+      // explicitly perform the interpolation to t_final as they do in the
+      // "normal" step mode.
+
+      ode_solver->Step(u, t, dt_real);
+
+      last_step = (t >= t_final - 1e-8*dt);
+
+      if (last_step || (ti % vis_steps) == 0)
+      {
+         if (myid == 0)
+         {
+            cout << "step " << ti << ", t = " << t << endl;
+         }
+
+         u_gf.SetFromTrueDofs(u);
+         if (visualization)
+         {
+            sout << "parallel " << num_procs << " " << myid << "\n";
+            sout << "solution\n" << *pmesh << u_gf << flush;
+         }
+
+         if (visit)
+         {
+            visit_dc.SetCycle(ti);
+            visit_dc.SetTime(t);
+            visit_dc.Save();
+         }
+      }
+      oper.SetParameters(u);
+   }
+   tic_toc.Stop();
+   if (myid == 0)
+   {
+      cout << "Done, " << tic_toc.RealTime() << "s." << endl;
+   }
+
+   // 11. Save the final solution in parallel. This output can be viewed later
+   //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
+   {
+      ostringstream sol_name;
+      sol_name << "ex16-final." << setfill('0') << setw(6) << myid;
+      ofstream osol(sol_name.str().c_str());
+      osol.precision(precision);
+      u_gf.Save(osol);
+   }
+
+   // 12. Free the used memory.
+   delete ode_solver;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
+                                       double kap, const Vector &u)
+   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
+     T(NULL),
+     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
+{
+   const double rel_tol = 1e-8;
+
+   M = new ParBilinearForm(&fespace);
+   M->AddDomainIntegrator(new MassIntegrator());
+   M->Assemble(0); // keep sparsity pattern of M and K the same
+   M->FormSystemMatrix(ess_tdof_list, Mmat);
+
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(rel_tol);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(100);
+   M_solver.SetPrintLevel(0);
+   M_prec.SetType(HypreSmoother::Jacobi);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(Mmat);
+
+   alpha = al;
+   kappa = kap;
+
+   T_solver.iterative_mode = false;
+   T_solver.SetRelTol(rel_tol);
+   T_solver.SetAbsTol(0.0);
+   T_solver.SetMaxIter(100);
+   T_solver.SetPrintLevel(0);
+   T_solver.SetPreconditioner(T_prec);
+
+   SetParameters(u);
+}
+
+void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
+{
+   // Compute:
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
+   Kmat.Mult(u, z);
+   z.Neg(); // z = -z
+   M_solver.Mult(z, du_dt);
+}
+
+void ConductionOperator::ImplicitSolve(const double dt,
+                                       const Vector &u, Vector &du_dt)
+{
+   // Solve the equation:
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, dt, Kmat);
+   T_solver.SetOperator(*T);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
+}
+
+int ConductionOperator::SUNImplicitSetup(const Vector &x,
+                                         const Vector &fx, int jok, int *jcur,
+                                         double gamma)
+{
+   // Setup the ODE Jacobian T = M + gamma K.
+   if (T) { delete T; }
+   T = Add(1.0, Mmat, gamma, Kmat);
+   T_solver.SetOperator(*T);
+   *jcur = 1;
+   return (0);
+}
+
+int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
+{
+   // Solve the system A x = z => (M - gamma K) x = M b.
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, x);
+   return (0);
+}
+
+void ConductionOperator::SetParameters(const Vector &u)
+{
+   ParGridFunction u_alpha_gf(&fespace);
+   u_alpha_gf.SetFromTrueDofs(u);
+   for (int i = 0; i < u_alpha_gf.Size(); i++)
+   {
+      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
+   }
+
+   delete K;
+   K = new ParBilinearForm(&fespace);
+
+   GridFunctionCoefficient u_coeff(&u_alpha_gf);
+
+   K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
+   K->Assemble(0); // keep sparsity pattern of M and K the same
+   K->FormSystemMatrix(ess_tdof_list, Kmat);
+}
+
+ConductionOperator::~ConductionOperator()
+{
+   delete T;
+   delete M;
+   delete K;
+}
+
+double InitialTemperature(const Vector &x)
+{
+   if (x.Norml2() < 0.5)
+   {
+      return 2.0;
+   }
+   else
+   {
+      return 1.0;
+   }
+}

examples/epic/makefile

@@ -0,0 +1,76 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/epic/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ex16
+PAR_EXAMPLES = ex16p
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_EPIC),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with EPIC)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+SERIAL_NAME := Serial EPIC example
+PARALLEL_NAME := Parallel EPIC example
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
+%-test-seq: %
+	@$(call mfem-test,$<,, $(SERIAL_NAME))
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f deformed.* velocity.* elastic_energy.*
+	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.* Example16*

examples/ex13p.cpp

@@ -55,6 +55,7 @@ int main(int argc, char *argv[])
    int order = 1;
    int nev = 5;
    bool visualization = 1;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -71,6 +72,8 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -86,13 +89,18 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement (2 by default, or
    //    specified on the command line with -rs).
    for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -100,7 +108,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution (1 time by
    //    default, or specified on the command line with -rp). Once the parallel
    //    mesh is defined, the serial mesh can be deleted.
@@ -112,7 +120,7 @@ int main(int argc, char *argv[])
    }
    pmesh->ReorientTetMesh();
 
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -122,7 +130,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
    //    element space. The first corresponds to the curl curl, while the second
    //    is a simple mass matrix needed on the right hand side of the
    //    generalized eigenvalue problem below. The boundary conditions are
@@ -164,7 +172,7 @@ int main(int argc, char *argv[])
    delete a;
    delete m;
 
-   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
    //    A to be used within the solver. Set the matrices which define the
    //    generalized eigenproblem A x = lambda M x.
    HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -180,15 +188,15 @@ int main(int argc, char *argv[])
    ame->SetMassMatrix(*M);
    ame->SetOperator(*A);
 
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
+   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //     parallel grid function to represent each of the eigenmodes returned by
+   //     the solver.
    Array<double> eigenvalues;
    ame->Solve();
    ame->GetEigenvalues(eigenvalues);
    ParGridFunction x(fespace);
 
-   // 10. Save the refined mesh and the modes in parallel. This output can be
+   // 11. Save the refined mesh and the modes in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
    {
       ostringstream mesh_name, mode_name;
@@ -213,7 +221,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 11. Send the solution by socket to a GLVis server.
+   // 12. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -253,7 +261,7 @@ int main(int argc, char *argv[])
       mode_sock.close();
    }
 
-   // 12. Free the used memory.
+   // 13. Free the used memory.
    delete ame;
    delete ams;
    delete M;

examples/ex19p.cpp

@@ -196,6 +196,12 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
    // 1. Initialize MPI
    MPI_Session mpi;
    const int myid = mpi.WorldRank();
@@ -438,16 +444,20 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
    // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
-
-   Vector disp_out(y.GetData() + block_trueOffsets[0],
+   Vector disp_in;
+   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
                    block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
+   Vector pres_in;
+   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
                    block_trueOffsets[2]-block_trueOffsets[1]);
 
+   Vector disp_out;
+   disp_out.MakeRef(y, block_trueOffsets[0],
+                    block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out;
+   pres_out.MakeRef(y, block_trueOffsets[1],
+                    block_trueOffsets[2]-block_trueOffsets[1]);
+
    Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
    Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
 
@@ -459,6 +469,9 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
    subtract(disp_in, temp, temp2);
 
    stiff_pcg->Mult(temp2, disp_out);
+
+   disp_out.SyncAliasMemory(y);
+   pres_out.SyncAliasMemory(y);
 }
 
 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -473,7 +486,10 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
 
       if (!spaces[0]->GetParMesh()->Nonconforming())
       {
+#ifndef HYPRE_USING_CUDA
+         // Not available yet when hypre is built with CUDA
          stiff_prec_amg->SetElasticityOptions(spaces[0]);
+#endif
       }
 
       stiff_prec = stiff_prec_amg;

examples/ex1p.cpp

@@ -89,7 +89,8 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
+                  "-no-a", "--no-algebraic",
                   "Use algebraic Ceed solver");
 #endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -197,15 +198,15 @@ int main(int argc, char *argv[])
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
    b.Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
    ParGridFunction x(&fespace);
    x = 0.0;
 
    // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));

examples/ex28p.cpp

@@ -81,6 +81,12 @@ Mesh * build_trapezoid_mesh(double offset)
 
 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
    // 1. Initialize MPI.
    int num_procs, myid;
    MPI_Init(&argc, &argv);
@@ -360,6 +366,7 @@ int main(int argc, char *argv[])
    }
    delete pmesh;
 
+   // HYPRE_Finalize();
    MPI_Finalize();
 
    return 0;

examples/ex2p.cpp

@@ -61,6 +61,7 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool amg_elast = 0;
    bool reorder_space = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -78,6 +79,8 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                   "Use byNODES ordering of vector space instead of byVDIM");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -93,7 +96,12 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -109,14 +117,14 @@ int main(int argc, char *argv[])
       return 3;
    }
 
-   // 4. Select the order of the finite element discretization space. For NURBS
+   // 5. Select the order of the finite element discretization space. For NURBS
    //    meshes, we increase the order by degree elevation.
    if (mesh->NURBSext)
    {
       mesh->DegreeElevate(order, order);
    }
 
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 6. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
@@ -129,7 +137,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -142,7 +150,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
    //    use vector finite elements, i.e. dim copies of a scalar finite element
    //    space. We use the ordering by vector dimension (the last argument of
    //    the FiniteElementSpace constructor) which is expected in the systems
@@ -175,7 +183,7 @@ int main(int argc, char *argv[])
            << "Assembling: " << flush;
    }
 
-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 9. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined by
    //    marking only boundary attribute 1 from the mesh as essential and
    //    converting it to a list of true dofs.
@@ -184,14 +192,14 @@ int main(int argc, char *argv[])
    ess_bdr[0] = 1;
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system. In this case, b_i equals the
-   //    boundary integral of f*phi_i where f represents a "pull down" force on
-   //    the Neumann part of the boundary and phi_i are the basis functions in
-   //    the finite element fespace. The force is defined by the object f, which
-   //    is a vector of Coefficient objects. The fact that f is non-zero on
-   //    boundary attribute 2 is indicated by the use of piece-wise constants
-   //    coefficient for its last component.
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system. In this case, b_i equals the
+   //     boundary integral of f*phi_i where f represents a "pull down" force on
+   //     the Neumann part of the boundary and phi_i are the basis functions in
+   //     the finite element fespace. The force is defined by the object f, which
+   //     is a vector of Coefficient objects. The fact that f is non-zero on
+   //     boundary attribute 2 is indicated by the use of piece-wise constants
+   //     coefficient for its last component.
    VectorArrayCoefficient f(dim);
    for (int i = 0; i < dim-1; i++)
    {
@@ -212,13 +220,13 @@ int main(int argc, char *argv[])
    }
    b->Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid
+   // 11. Define the solution vector x as a parallel finite element grid
    //     function corresponding to fespace. Initialize x with initial guess of
    //     zero, which satisfies the boundary conditions.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
    //     corresponding to the linear elasticity integrator with piece-wise
    //     constants coefficient lambda and mu.
    Vector lambda(pmesh->attributes.Max());
@@ -233,7 +241,7 @@ int main(int argc, char *argv[])
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
 
-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 13. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, static condensation, etc.
@@ -250,7 +258,7 @@ int main(int argc, char *argv[])
       cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
    }
 
-   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
    //     preconditioner from hypre.
    HypreBoomerAMG *amg = new HypreBoomerAMG(A);
    if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -268,11 +276,11 @@ int main(int argc, char *argv[])
    pcg->SetPreconditioner(*amg);
    pcg->Mult(B, X);
 
-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 15. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
    //     space. This means that we define the mesh elements through a fespace
    //     based transformation of the reference element.  This allows us to save
    //     the displaced mesh as a curved mesh when using high-order finite
@@ -284,7 +292,7 @@ int main(int argc, char *argv[])
       pmesh->SetNodalFESpace(fespace);
    }
 
-   // 16. Save in parallel the displaced mesh and the inverted solution (which
+   // 17. Save in parallel the displaced mesh and the inverted solution (which
    //     gives the backward displacements to the original grid). This output
    //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
@@ -305,7 +313,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
    //     keys in GLVis to visualize the displacements.
    if (visualization)
    {
@@ -317,7 +325,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 18. Free the used memory.
+   // 19. Free the used memory.
    delete pcg;
    delete amg;
    delete a;

examples/ex3p.cpp

@@ -103,6 +103,7 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
+      // HYPRE_Finalize();
       MPI_Finalize();
       return 1;
    }

examples/ex5.cpp

@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
+      if (Device::IsEnabled()) { B.BuildTranspose(); }
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);
@@ -240,6 +241,7 @@ int main(int argc, char *argv[])
    {
       SparseMatrix &M(mVarf->SpMat());
       M.GetDiag(Md);
+      Md.HostReadWrite();
 
       SparseMatrix &B(bVarf->SpMat());
       MinvBt = Transpose(B);
@@ -287,12 +289,18 @@ int main(int argc, char *argv[])
    chrono.Stop();
 
    if (solver.GetConverged())
+   {
       std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
+                << " iterations with a residual norm of "
+                << solver.GetFinalNorm() << ".\n";
+   }
    else
+   {
       std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm()
+                << ".\n";
+   }
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
 
    // 12. Create the grid functions u and p. Compute the L2 error norms.
    GridFunction u, p;

examples/ex7p.cpp

@@ -47,6 +47,7 @@ int main(int argc, char *argv[])
    int order = 2;
    bool always_snap = false;
    bool visualization = 1;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&elem_type, "-e", "--elem",
@@ -65,6 +66,8 @@ int main(int argc, char *argv[])
                   "--snap-at-the-end",
                   "If true, snap nodes to the sphere initially and after each refinement "
                   "otherwise, snap only after the last refinement");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -80,7 +83,12 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
    //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
    //    the elements and the vertices of the mesh, and then make it high-order
    //    by specifying a finite element space for its nodes.
@@ -146,7 +154,7 @@ int main(int argc, char *argv[])
    FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
    mesh->SetNodalFESpace(&nodal_fes);
 
-   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
    //    refinements is fixed to 2.
    for (int l = 0; l <= ref_levels; l++)
    {
@@ -218,7 +226,7 @@ int main(int argc, char *argv[])
       SnapNodes(*pmesh);
    }
 
-   // 5. Define a finite element space on the mesh. Here we use isoparametric
+   // 6. Define a finite element space on the mesh. Here we use isoparametric
    //    finite elements -- the same as the mesh nodes.
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
    HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -227,7 +235,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
    //    the basis functions in the finite element fespace.
    ParLinearForm *b = new ParLinearForm(fespace);
@@ -237,27 +245,27 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
    b->Assemble();
 
-   // 7. Define the solution vector x as a finite element grid function
+   // 8. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 8. Set up the bilinear form a(.,.) on the finite element space
+   // 9. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    and Mass domain integrators.
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
    a->AddDomainIntegrator(new MassIntegrator(one));
 
-   // 9. Assemble the parallel linear system, applying any transformations
-   //    such as: parallel assembly, applying conforming constraints, etc.
+   // 10. Assemble the parallel linear system, applying any transformations
+   //     such as: parallel assembly, applying conforming constraints, etc.
    a->Assemble();
    HypreParMatrix A;
    Vector B, X;
    Array<int> empty_tdof_list;
    a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);
 
-   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
    //     preconditioner from hypre. Extract the parallel grid function x
    //     corresponding to the finite element approximation X. This is the local
    //     solution on each processor.
@@ -273,14 +281,14 @@ int main(int argc, char *argv[])
    delete a;
    delete b;
 
-   // 11. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
    double err = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
       cout << "\nL2 norm of error: " << err << endl;
    }
 
-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
    {
       ostringstream mesh_name, sol_name;
@@ -296,7 +304,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -307,7 +315,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 14. Free the used memory.
+   // 15. Free the used memory.
    delete pcg;
    delete amg;
    delete fespace;

examples/makefile

@@ -26,6 +26,9 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
+  ex24p ex25p ex26p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -42,6 +45,9 @@ endif
 ifeq ($(MFEM_USE_HIOP),YES)
    SUBDIRS += hiop
 endif
+ifeq ($(MFEM_USE_EPIC),YES)
+   SUBDIRS += epic
+endif
 ifeq ($(MFEM_USE_PETSC),YES)
    SUBDIRS += petsc
 endif
@@ -99,6 +105,14 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, Serial CUDA example,-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, Serial HIP example,-d hip)
 
 # Testing: Specific execution options
 ex0-test-seq: ex0

examples/superlu/ex1p.cpp

@@ -282,6 +282,10 @@ int main(int argc, char *argv[])
    superlu->SetOperator(*SLU_A);
    superlu->SetPrintStatistics(true);
    superlu->Mult(B, X);
+   superlu->DismantleGrid();
+
+   delete SLU_A;
+   delete superlu;
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.

fem/CMakeLists.txt

@@ -184,6 +184,11 @@ if (MFEM_USE_ADIOS2)
   list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
+if (MFEM_USE_FMS)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+endif()
+
 if (MFEM_USE_MPI)
   list(APPEND SRCS
     pbilinearform.cpp

fem/bilinearform.cpp

@@ -725,8 +725,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       {
          // A, X and B point to the same data as mat, x and b
          EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.MakeRef(x, 0, x.Size());
+         B.MakeRef(b, 0, b.Size());
          if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
       }
    }

fem/bilininteg.hpp

@@ -711,7 +711,7 @@ protected:
    {
       return "MixedScalarDerivativeIntegrator:  "
              "Trial and test spaces must both be scalar fields in 1D "
-             "and the trial space must implement CaldDShape.";
+             "and the trial space must implement CalcDShape.";
    }
 
    inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
@@ -2936,11 +2936,11 @@ public:
 
     - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
       compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
-      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77-88. Springer
       Berlin Heidelberg, 2000.
     - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
       of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
-      Numerical Analysis, 39(5):1749–1779, 2002.
+      Numerical Analysis, 39(5):1749-1779, 2002.
 */
 class DGDiffusionBR2Integrator : public BilinearFormIntegrator
 {

fem/complex_fem.cpp

@@ -1204,17 +1204,30 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       });
       // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      ess_tdof_list.HostRead();
       if (A_i.Type() == Operator::Hypre_ParCSR)
       {
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
+#ifndef HYPRE_USING_CUDA
+         ess_tdof_list.HostRead();
          for (int k = 0; k < n; k++)
          {
             const int j = ess_tdof_list[k];
             Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
+#else
+         Ah->HypreReadWrite();
+         const int *d_ess_tdof_list =
+            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+         const int *d_diag_i = Aih->diag->i;
+         double *d_diag_data = Aih->diag->data;
+         CuWrap1D(n, [=] MFEM_DEVICE (int k)
+         {
+            const int j = d_ess_tdof_list[k];
+            d_diag_data[d_diag_i[j]] = 0.0;
+         });
+#endif
       }
       else
       {

fem/eltrans.hpp

@@ -316,7 +316,7 @@ public:
 
    /// Set the desired print level, useful for debugging.
    /** The valid options are: -1 - never print (default); 0 - print only errors;
-       1 - print the first and last last iterations; 2 - print every iteration;
+       1 - print the first and last iterations; 2 - print every iteration;
        and 3 - print every iteration including point coordinates. */
    void SetPrintLevel(int pr_level) { print_level = pr_level; }
 

fem/fe.cpp

@@ -495,6 +495,7 @@ void ScalarFiniteElement::ScalarLocalRestriction(
       R *= 1.0 / Trans.Weight();
    }
 }
+
 const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                                    DofToQuad::Mode mode) const
 {

fem/fe.hpp

@@ -97,7 +97,7 @@ public:
       {
          "Gauss-Legendre", "Gauss-Lobatto", "Positive (Bernstein)",
          "Open uniform", "Closed uniform", "Open half uniform",
-         "Seredipity", "Closed Gauss-Legendre",
+         "Serendipity", "Closed Gauss-Legendre",
          "Integrated Gauss-Lobatto indicator"
       };
       return name[Check(b_type)];
@@ -1126,7 +1126,7 @@ public:
    { dofs = 1.0; }
 };
 
-/// A 1D quadractic finite element with uniformly spaced nodes
+/// A 1D quadratic finite element with uniformly spaced nodes
 class Quad1DFiniteElement : public NodalFiniteElement
 {
 public:

fem/fem.hpp

@@ -64,4 +64,9 @@
 #include "adios2datacollection.hpp"
 #endif
 
+#ifdef MFEM_USE_FMS
+#include "fmsconvert.hpp"
+#include "fmsdatacollection.hpp"
+#endif
+
 #endif

fem/fmsconvert.hpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef FMS_CONVERT
+#define FMS_CONVERT
+
+#include "../config/config.hpp"
+#include "datacollection.hpp"
+
+#ifdef MFEM_USE_FMS
+#include <fms.h>
+
+namespace mfem
+{
+
+/** In-memory conversion of FMS data collection to an MFEM data collection.
+    @param dc The FMS data collection to convert.
+    @param[out] mfem_dc A pointer to a new MFEM DataCollection containing the
+    FMS data.
+    @return 0 on success; non-zero on failure.
+*/
+int FmsDataCollectionToDataCollection(FmsDataCollection dc,
+                                      DataCollection **mfem_dc);
+
+/** In-memory conversion of MFEM data collection to an FMS data collection.
+    @param mfem_dc The MFEM data collection to convert.
+    @param[out] dc A pointer to a new FmsDataCollection containing the MFEM
+    data.
+    @return 0 on success; non-zero on failure.
+*/
+int DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
+                                      FmsDataCollection *dc);
+
+} // namespace mfem
+
+#endif
+
+#endif

fem/fmsdatacollection.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "fem.hpp"
+#include "../general/text.hpp"
+
+#include <fmsio.h>
+
+#include <string>
+#include <sstream>
+
+namespace mfem
+{
+
+// class FMSDataCollection implementation
+
+FMSDataCollection::FMSDataCollection(const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   appendRankToFileName = false; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#ifdef MFEM_USE_MPI
+
+FMSDataCollection::FMSDataCollection(MPI_Comm comm,
+                                     const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   m_comm = comm;
+   MPI_Comm_rank(comm, &myid);
+   MPI_Comm_size(comm, &num_procs);
+   appendRankToFileName = true; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#endif
+
+FMSDataCollection::~FMSDataCollection()
+{
+   // empty
+}
+
+void FMSDataCollection::Save()
+{
+   // Convert this to FmsDataCollection.
+
+   FmsDataCollection dc;
+   if (DataCollectionToFmsDataCollection(this, &dc) == 0)
+   {
+      std::string root(RootFileName());
+      int err = FmsIOWrite(root.c_str(), fms_protocol.c_str(), dc);
+      FmsDataCollectionDestroy(&dc);
+      if (err)
+      {
+         MFEM_ABORT("Error creating FMS file: " << root);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Error converting data collection");
+   }
+}
+
+void FMSDataCollection::Load(int cycle)
+{
+   DeleteAll();
+   this->cycle = cycle;
+
+   FmsDataCollection dc;
+   std::string root(RootFileName());
+   int err = FmsIORead(root.c_str(), fms_protocol.c_str(), &dc);
+
+   if (err == 0)
+   {
+      DataCollection *mdc = nullptr;
+      if (FmsDataCollectionToDataCollection(dc,&mdc) == 0)
+      {
+         // Tell the data collection we read that it does not own data.
+         // We will steal its data.
+         mdc->SetOwnData(false);
+
+         SetCycle(mdc->GetCycle());
+         SetTime(mdc->GetTime());
+         SetTimeStep(mdc->GetTimeStep());
+         name = mdc->GetCollectionName();
+
+         // Set mdc's mesh as our mesh.
+         SetMesh(mdc->GetMesh());
+
+         // Set mdc's fields/qfields as ours.
+         std::vector<std::string> names;
+         for (const auto &pair : mdc->GetFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         names.clear();
+         for (const auto &pair : mdc->GetQFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterQField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         // Indicate that we own the data.
+         SetOwnData(true);
+
+         // Delete mdc. We stole its contents.
+         delete mdc;
+      }
+      FmsDataCollectionDestroy(&dc);
+   }
+   else
+   {
+      MFEM_ABORT("Error reading data collection: " << root);
+   }
+}
+
+void FMSDataCollection::SetProtocol(const std::string &protocol)
+{
+   fms_protocol = protocol;
+}
+
+std::string FMSDataCollection::RootFileName()
+{
+   std::string res;
+   if (pad_digits_cycle)
+   {
+      res = prefix_path + name + "_" +
+            to_padded_string(cycle, pad_digits_cycle) +
+            ".fms";
+   }
+   else
+   {
+      res = prefix_path + name + ".fms";
+   }
+   return res;
+}
+
+} // namespace mfem
+
+#endif

fem/fmsdatacollection.hpp

@@ -0,0 +1,74 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FMSDATACOLLECTION
+#define MFEM_FMSDATACOLLECTION
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "datacollection.hpp"
+#include <fms.h>
+
+namespace mfem
+{
+
+/** @brief Data collection that uses FMS. */
+/** FMSDataCollection lets MFEM read/write data using FMS.
+
+    For more information, see:
+    - FMS project, https://ceed.exascaleproject.org/fms/
+*/
+
+/// Data collection with FMS I/O routines
+class FMSDataCollection : public DataCollection
+{
+protected:
+   // file name helpers
+
+   /// Returns file name for the current cycle
+   std::string RootFileName();
+
+   // holds currently active i/o protocol
+   std::string fms_protocol;
+
+public:
+   /// Constructor. The collection name is used when saving the data.
+   /** If @a mesh is NULL, then the mesh can be set later by calling either
+       SetMesh() or Load(). The latter works only in serial. */
+   FMSDataCollection(const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#ifdef MFEM_USE_MPI
+   /// Construct a parallel FMSDataCollection.
+   FMSDataCollection(MPI_Comm comm, const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#endif
+
+   /// We will delete the mesh and fields if we own them
+   virtual ~FMSDataCollection();
+
+   /// Set the FMS relay i/o protocol to use
+   /** Supported options: ascii (default), json, yaml, hdf5 */
+   void SetProtocol(const std::string &protocol);
+
+   /// Save the collection and a FMS blueprint root file
+   virtual void Save();
+
+   /// Load the collection based blueprint data
+   virtual void Load(int cycle = 0);
+};
+
+} // namespace mfem
+
+#endif
+
+#endif

fem/gridfunc.cpp

@@ -218,7 +218,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 {
-   if (!f->GetProlongationMatrix())
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
    {
       MakeRef(f, tv);
       t_vec.NewDataAndSize(tv, size);
@@ -232,7 +232,8 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
 {
-   if (!f->GetProlongationMatrix())
+   tv.UseDevice(true);
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
    {
       MakeRef(f, tv, tv_offset);
       t_vec.NewMemoryAndSize(data, size, false);
@@ -241,10 +242,7 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
    {
       MFEM_ASSERT(tv.Size() >= tv_offset + f->GetTrueVSize(), "");
       SetSpace(f); // works in parallel
-      tv.UseDevice(true);
-      const int tv_size = f->GetTrueVSize();
-      t_vec.NewMemoryAndSize(Memory<double>(tv.GetMemory(), tv_offset, tv_size),
-                             tv_size, true);
+      t_vec.MakeRef(tv, tv_offset, f->GetTrueVSize());
    }
 }
 
@@ -334,10 +332,10 @@ int GridFunction::VectorDim() const
 void GridFunction::GetTrueDofs(Vector &tv) const
 {
    const SparseMatrix *R = fes->GetRestrictionMatrix();
-   if (!R)
+   if (!R || IsIdentityProlongation(fes->GetProlongationMatrix()))
    {
-      // R is identity -> make tv a reference to *this
-      tv.MakeRef(const_cast<GridFunction &>(*this), 0, size);
+      // R is identity
+      tv = *this; // no real copy if 'tv' and '*this' use the same data
    }
    else
    {

fem/gridfunc.hpp

@@ -130,9 +130,7 @@ public:
        or set. */
    Vector &GetTrueVector() { return t_vec; }
 
-   /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
-   /// then `tv` will be set to point to the data of `*this`.
-   /** @warning This method breaks const-ness when all dofs are true. */
+   /// Extract the true-dofs from the GridFunction.
    void GetTrueDofs(Vector &tv) const;
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.

fem/gslib.cpp

@@ -19,7 +19,11 @@
 #pragma GCC diagnostic ignored "-Wunused-function"
 #endif
 
+// External GSLIB header (the MFEM header is gslib.hpp)
+namespace gslib
+{
 #include "gslib.h"
+}
 
 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
 #pragma GCC diagnostic pop
@@ -34,13 +38,13 @@ FindPointsGSLIB::FindPointsGSLIB()
      dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr       = new crystal;
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
 #ifdef MFEM_USE_MPI
    int initialized;
    MPI_Initialized(&initialized);
    if (!initialized) { MPI_Init(NULL, NULL); }
-   MPI_Comm comm = MPI_COMM_WORLD;;
+   MPI_Comm comm = MPI_COMM_WORLD;
    comm_init(gsl_comm, comm);
 #else
    comm_init(gsl_comm, 0);
@@ -62,8 +66,8 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
      dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr      = new crystal;
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
    comm_init(gsl_comm, comm_);
 }
 #endif
@@ -728,7 +732,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
       }
 
       // Pack data to send via crystal router
-      struct array *outpt = new array;
+      struct gslib::array *outpt = new gslib::array;
       struct out_pt { double r[3], ival; uint index, el, proc; };
       struct out_pt *pt;
       array_init(struct out_pt, outpt, nptsend);
@@ -788,7 +792,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          }
 
          // Save index and proc data in a struct
-         struct array *savpt = new array;
+         struct gslib::array *savpt = new gslib::array;
          struct sav_pt { uint index, proc; };
          struct sav_pt *spt;
          array_init(struct sav_pt, savpt, npt);
@@ -806,7 +810,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          delete outpt;
 
          // Copy data from save struct to send struct and send component wise
-         struct array *sendpt = new array;
+         struct gslib::array *sendpt = new gslib::array;
          struct send_pt { double ival; uint index, proc; };
          struct send_pt *sdpt;
          for (int j = 0; j < ncomp; j++)

fem/gslib.hpp

@@ -17,11 +17,13 @@
 
 #ifdef MFEM_USE_GSLIB
 
+namespace gslib
+{
 struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
-struct array;
 struct crystal;
+}
 
 namespace mfem
 {
@@ -50,10 +52,10 @@ public:
 protected:
    Mesh *mesh, *meshsplit;
    IntegrationRule *ir_simplex;    // IntegrationRule to split quads/hex -> simplex
-   struct findpts_data_2 *fdata2D; // gslib's internal data
-   struct findpts_data_3 *fdata3D; // gslib's internal data
-   struct crystal *cr;             // gslib's internal data
-   struct comm *gsl_comm;          // gslib's internal data
+   struct gslib::findpts_data_2 *fdata2D; // gslib's internal data
+   struct gslib::findpts_data_3 *fdata3D; // gslib's internal data
+   struct gslib::crystal *cr;             // gslib's internal data
+   struct gslib::comm *gsl_comm;          // gslib's internal data
    int dim, points_cnt;
    Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
    Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;

fem/linearform.cpp

@@ -267,18 +267,16 @@ void LinearForm::Assemble()
 
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
+   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
    fes = f;
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, f->GetVSize()),
-                    f->GetVSize(), false);
+   v.UseDevice(true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    ResetDeltaLocations();
 }
 
 void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 {
-   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
-   fes = f;
-   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+   Update(f, v, v_offset);
 }
 
 void LinearForm::AssembleDelta()

fem/nonlinearform.cpp

@@ -630,7 +630,7 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
 
 double BlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs.Update(x.GetData(), block_offsets);
+   xs.Update(const_cast<Vector&>(x), block_offsets);
    return GetEnergyBlocked(xs);
 }
 
@@ -646,7 +646,9 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
    Array<const FiniteElement *> fe2(fes.Size());
    ElementTransformation *T;
 
+   by.UseDevice(true);
    by = 0.0;
+   by.SyncToBlocks();
    for (int s=0; s<fes.Size(); ++s)
    {
       el_x_const[s] = el_x[s] = new Vector();
@@ -785,6 +787,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
       delete el_y[s];
       delete el_x[s];
    }
+
+   by.SyncFromBlocks();
 }
 
 const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
@@ -805,8 +809,8 @@ const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
 
 void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
-   BlockVector by(y.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
+   BlockVector by(y, block_trueOffsets);
 
    const BlockVector &pbx = Prolongate(bx);
    if (needs_prolongation)
@@ -815,8 +819,8 @@ void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
    }
    BlockVector &pby = needs_prolongation ? aux2 : by;
 
-   xs.Update(pbx.GetData(), block_offsets);
-   ys.Update(pby.GetData(), block_offsets);
+   xs.Update(const_cast<BlockVector&>(pbx), block_offsets);
+   ys.Update(pby, block_offsets);
    MultBlocked(xs, ys);
 
    for (int s = 0; s < fes.Size(); s++)
@@ -1021,7 +1025,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
 
 Operator &BlockNonlinearForm::GetGradient(const Vector &x) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
    const BlockVector &pbx = Prolongate(bx);
 
    ComputeGradientBlocked(pbx);

fem/nonlininteg.hpp

@@ -121,7 +121,7 @@ public:
        @param[in,out] y  The result Vector: @f$ y += G x @f$. */
    virtual void AddMultGradPA(const Vector &x, Vector &y) const;
 
-   /// Method for computing the diagonal of the gradient with partial assmebly.
+   /// Method for computing the diagonal of the gradient with partial assembly.
    /** The result Vector @a diag is an E-Vector. This method can be called only
        after the method AssembleGradPA() has been called.
 

fem/pbilinearform.cpp

@@ -473,7 +473,7 @@ void ParBilinearForm::RecoverFEMSolution(
    else
    {
       // Apply conforming prolongation
-      x.SetSize(P.Height());
+      x.SetSize(P.Height(), GetHypreMemoryType());
       P.Mult(X, x);
    }
 }

fem/pfespace.cpp

@@ -2885,19 +2885,10 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
 
    HypreParMatrix* R;
    R = new HypreParMatrix(MyComm, dof_offsets[nrk], old_dof_offsets[nrk],
-                          dof_offsets, old_dof_offsets, diag, offd, cmap);
+                          dof_offsets, old_dof_offsets, diag, offd, cmap,
+                          true);
 
-#ifndef HYPRE_BIGINT
-   diag->LoseData();
-   offd->LoseData();
-#else
-   diag->SetDataOwner(false);
-   offd->SetDataOwner(false);
-#endif
-   delete diag;
-   delete offd;
-
-   R->SetOwnerFlags(3, 3, 1);
+   R->SetOwnerFlags(R->OwnsDiag(), R->OwnsOffd(), 1);
 
    return R;
 }

fem/pfespace.hpp

@@ -291,7 +291,7 @@ public:
    /** Returns pointer to the FiniteElement in the FiniteElementCollection
        associated with i'th element in the mesh object. If @a i is greater than
        or equal to the number of local mesh elements, @a i will be interpreted
-       as a shifted index of a face neigbor element. */
+       as a shifted index of a face neighbor element. */
    virtual const FiniteElement *GetFE(int i) const;
 
    /** Returns an Operator that converts L-vectors to E-vectors on each face.

fem/pnonlinearform.cpp

@@ -218,7 +218,8 @@ void ParBlockNonlinearForm::SetEssentialBC(const
 
 double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s = 0; s < fes.Size(); ++s)
@@ -237,8 +238,9 @@ double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 
 void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
-   ys_true.Update(y.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
+   ys_true.Update(y, block_trueOffsets);
    xs.Update(block_offsets);
    ys.Update(block_offsets);
 
@@ -262,13 +264,17 @@ void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 
       ys_true.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
    }
+
+   ys_true.SyncFromBlocks();
+   y.SyncMemory(ys_true);
 }
 
 /// Return the local gradient matrix for the given true-dof vector x
 const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
    const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s=0; s<fes.Size(); ++s)
@@ -277,7 +283,8 @@ const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
          xs_true.GetBlock(s), xs.GetBlock(s));
    }
 
-   BlockNonlinearForm::ComputeGradientBlocked(xs); // (re)assemble Grad with b.c.
+   // (re)assemble Grad without b.c. into 'Grads'
+   BlockNonlinearForm::ComputeGradientBlocked(xs);
 
    delete BlockGrad;
    BlockGrad = new BlockOperator(block_offsets);

fem/restriction.cpp

@@ -13,6 +13,7 @@
 #include "gridfunc.hpp"
 #include "fespace.hpp"
 #include "../general/forall.hpp"
+#include <climits>
 
 namespace mfem
 {
@@ -267,35 +268,25 @@ void ElementRestriction::FillSparseMatrix(const Vector &mat_ea,
    FillJAndData(mat_ea, mat);
 }
 
-template <int MaxNbNbr>
 static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
                                       const int *nbr_elts, const int nbrNbElts)
 {
-   // Building the intersection
-   int inter[MaxNbNbr];
-   int cpt = 0;
+   // Find the minimal element index found in both my_elts[] and nbr_elts[]
+   int min_el = INT_MAX;
    for (int i = 0; i < nbElts; i++)
    {
       const int e_i = my_elts[i];
+      if (e_i >= min_el) { continue; }
       for (int j = 0; j < nbrNbElts; j++)
       {
          if (e_i==nbr_elts[j])
          {
-            inter[cpt] = e_i;
-            cpt++;
+            min_el = e_i; // we already know e_i < min_el
+            break;
          }
       }
    }
-   // Finding the minimum
-   int min = inter[0];
-   for (int i = 1; i < cpt; i++)
-   {
-      if (inter[i] < min)
-      {
-         min = inter[i];
-      }
-   }
-   return min;
+   return min_el;
 }
 
 /** Returns the index where a non-zero entry should be added and increment the
@@ -355,7 +346,7 @@ int ElementRestriction::FillI(SparseMatrix &mat) const
                   const int elt = j_E/elt_dofs;
                   j_elts[e_j] = elt;
                }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
                if (e == min_e) // add the nnz only once
                {
                   GetAndIncrementNnzIndex(i_L, I);
@@ -434,7 +425,7 @@ void ElementRestriction::FillJAndData(const Vector &ea_data,
                   j_elts[e_j] = elt;
                   j_B[e_j]    = j_E%elt_dofs;
                }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
                if (e == min_e) // add the nnz only once
                {
                   double val = 0.0;

general/array.hpp

@@ -339,6 +339,10 @@ inline bool operator!=(const Array<T> &LHS, const Array<T> &RHS)
 }
 
 
+/// Utility function similar to std::as_const in c++17.
+template <typename T> const T &AsConst(T &a) { return a; }
+
+
 template <class T>
 class Array2D;
 

general/error.cpp

@@ -92,7 +92,7 @@ void mfem_backtrace(int mode, int depth)
    int err = unw_getcontext(&uc);
    err = err ? err : unw_init_local(&cursor, &uc);
 
-   Array<unw_word_t> addrs;
+   Array<unw_word_t> addrs(MemoryType::HOST);
    while (unw_step(&cursor) > 0 && addrs.Size() != depth)
    {
       err = err ? err : unw_get_proc_name(&cursor, name, UNW_NAME_LEN, &offp);

general/mem_manager.cpp

@@ -45,6 +45,9 @@
 #endif
 #endif // MFEM_USE_UMPIRE
 
+// Internal debug option, useful for tracking some memory manager operations.
+// #define MFEM_TRACK_MEM_MANAGER
+
 namespace mfem
 {
 
@@ -63,6 +66,27 @@ MemoryType GetMemoryType(MemoryClass mc)
 }
 
 
+bool MemoryClassContainsType(MemoryClass mc, MemoryType mt)
+{
+   switch (mc)
+   {
+      case MemoryClass::HOST: return IsHostMemory(mt);
+      case MemoryClass::HOST_32:
+         return (mt == MemoryType::HOST_32 ||
+                 mt == MemoryType::HOST_64 ||
+                 mt == MemoryType::HOST_DEBUG);
+      case MemoryClass::HOST_64:
+         return (mt == MemoryType::HOST_64 ||
+                 mt == MemoryType::HOST_DEBUG);
+      case MemoryClass::DEVICE: return IsDeviceMemory(mt);
+      case MemoryClass::MANAGED:
+         return (mt == MemoryType::MANAGED);
+   }
+   MFEM_ABORT("invalid MemoryClass");
+   return false;
+}
+
+
 static void MFEM_VERIFY_TYPES(const MemoryType h_mt, const MemoryType d_mt)
 {
    MFEM_VERIFY(IsHostMemory(h_mt), "h_mt = " << (int)h_mt);
@@ -147,10 +171,12 @@ struct Memory
 /// Alias class that holds the base memory region and the offset
 struct Alias
 {
-   Memory *const mem;
-   const size_t offset, bytes;
+   Memory *mem;
+   size_t offset;
    size_t counter;
-   const MemoryType h_mt;
+   // 'h_mt' is already stored in 'mem', however, we use this field for type
+   // checking since the alias may be dangling, i.e. 'mem' may be invalid.
+   MemoryType h_mt;
 };
 
 /// Maps for the Memory and the Alias classes
@@ -558,7 +584,7 @@ public:
 #ifdef MFEM_USE_HIP
       return HipMemcpyHtoD(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
    void *DtoD(void* dst, const void* src, size_t bytes) override
    {
@@ -568,7 +594,7 @@ public:
 #ifdef MFEM_USE_HIP
       return HipMemcpyDtoD(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
    void *DtoH(void *dst, const void *src, size_t bytes) override
    {
@@ -578,7 +604,7 @@ public:
 #ifdef MFEM_USE_HIP
       return HipMemcpyDtoH(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
    }
 };
 #else
@@ -759,7 +785,7 @@ void *MemoryManager::New_(void *h_tmp, size_t bytes, MemoryType h_mt,
    // mm.InsertDevice(nullptr, h_ptr, bytes, h_mt, d_mt); // non-lazy dev alloc
 
    // MFEM_VERIFY_TYPES(h_mt, mt); // done by mm.Insert() above
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, false);
 
    return h_ptr;
 }
@@ -770,7 +796,7 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
 {
    MFEM_CONTRACT_VAR(alias);
    MFEM_ASSERT(exists, "Internal error!");
-   MFEM_ASSERT(!alias, "Cannot register an alias!");
+   MFEM_VERIFY(!alias, "Cannot register an alias!");
    const bool is_host_mem = IsHostMemory(mt);
    const MemType h_mt = is_host_mem ? mt : GetDualMemoryType(mt);
    const MemType d_mt = is_host_mem ? MemoryType::DEFAULT : mt;
@@ -798,20 +824,21 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
    }
    else // DEVICE TYPES
    {
-      MFEM_VERIFY(ptr, "cannot register NULL device pointer");
+      MFEM_VERIFY(ptr || bytes == 0,
+                  "cannot register NULL device pointer with bytes = " << bytes);
       if (h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
       else { h_ptr = h_tmp; }
       mm.InsertDevice(ptr, h_ptr, bytes, h_mt, d_mt);
       flags = own ? flags | Mem::OWNS_DEVICE : flags & ~Mem::OWNS_DEVICE;
       flags |= (Mem::OWNS_HOST | Mem::VALID_DEVICE);
    }
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, alias);
    return h_ptr;
 }
 
-void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,
-                              MemoryType h_mt, MemoryType d_mt,
-                              bool own, bool alias, unsigned &flags)
+void MemoryManager::Register2_(void *h_ptr, void *d_ptr, size_t bytes,
+                               MemoryType h_mt, MemoryType d_mt,
+                               bool own, bool alias, unsigned &flags)
 {
    MFEM_CONTRACT_VAR(alias);
    MFEM_ASSERT(exists, "Internal error!");
@@ -826,12 +853,14 @@ void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,
 
    flags |= Mem::REGISTERED | Mem::OWNS_INTERNAL;
 
+   MFEM_VERIFY(d_ptr || bytes == 0,
+               "cannot register NULL device pointer with bytes = " << bytes);
    mm.InsertDevice(d_ptr, h_ptr, bytes, h_mt, d_mt);
    flags = (own ? flags | (Mem::OWNS_HOST | Mem::OWNS_DEVICE) :
             flags & ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE)) |
            Mem::VALID_HOST;
 
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, alias);
 }
 
 void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
@@ -871,37 +900,40 @@ void MemoryManager::SetDeviceMemoryType_(void *h_ptr, unsigned flags,
    }
 }
 
-MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType mt, unsigned flags)
+MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType h_mt, unsigned flags)
 {
    const bool alias = flags & Mem::ALIAS;
    const bool registered = flags & Mem::REGISTERED;
    const bool owns_host = flags & Mem::OWNS_HOST;
    const bool owns_device = flags & Mem::OWNS_DEVICE;
    const bool owns_internal = flags & Mem::OWNS_INTERNAL;
-   MFEM_ASSERT(registered || IsHostMemory(mt),"");
+   MFEM_ASSERT(IsHostMemory(h_mt), "invalid h_mt = " << (int)h_mt);
+   // MFEM_ASSERT(registered || IsHostMemory(h_mt),"");
    MFEM_ASSERT(!owns_device || owns_internal, "invalid Memory state");
-   if (!mm.exists || !registered) { return mt; }
+   MFEM_ASSERT(registered || !(owns_host || owns_device || owns_internal),
+               "invalid Memory state");
+   if (!mm.exists || !registered) { return h_mt; }
    if (alias)
    {
       if (owns_internal)
       {
-         const MemoryType h_mt = maps->aliases.at(h_ptr).h_mt;
-         MFEM_ASSERT(mt == h_mt,"");
+         MFEM_ASSERT(mm.IsAlias(h_ptr), "");
+         MFEM_ASSERT(h_mt == maps->aliases.at(h_ptr).h_mt, "");
          mm.EraseAlias(h_ptr);
-         return h_mt;
       }
    }
    else // Known
    {
-      const MemoryType h_mt = mt;
-      MFEM_ASSERT(!owns_internal ||
-                  mt == maps->memories.at(h_ptr).h_mt,"");
       if (owns_host && (h_mt != MemoryType::HOST))
       { ctrl->Host(h_mt)->Dealloc(h_ptr); }
-      if (owns_internal) { mm.Erase(h_ptr, owns_device); }
-      return h_mt;
+      if (owns_internal)
+      {
+         MFEM_ASSERT(mm.IsKnown(h_ptr), "");
+         MFEM_ASSERT(h_mt == maps->memories.at(h_ptr).h_mt, "");
+         mm.Erase(h_ptr, owns_device);
+      }
    }
-   return mt;
+   return h_mt;
 }
 
 void MemoryManager::DeleteDevice_(void *h_ptr, unsigned & flags)
@@ -923,15 +955,19 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
       MFEM_VERIFY(bytes == 0, "Trying to access NULL with size " << bytes);
       return true;
    }
-
-   const bool known = mm.IsKnown(h_ptr);
-   const bool alias = mm.IsAlias(h_ptr);
-   const bool check = known || ((flags & Mem::ALIAS) && alias);
-   MFEM_VERIFY(check, "Unknown host pointer: " << h_ptr);
-   const internal::Memory &mem =
-      (flags & Mem::ALIAS) ?
-      *maps->aliases.at(h_ptr).mem : maps->memories.at(h_ptr);
-   MemoryType d_mt = mem.d_mt;
+   MemoryType d_mt;
+   if (!(flags & Mem::ALIAS))
+   {
+      auto iter = maps->memories.find(h_ptr);
+      MFEM_VERIFY(iter != maps->memories.end(), "internal error");
+      d_mt = iter->second.d_mt;
+   }
+   else
+   {
+      auto iter = maps->aliases.find(h_ptr);
+      MFEM_VERIFY(iter != maps->aliases.end(), "internal error");
+      d_mt = iter->second.mem->d_mt;
+   }
    if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
    switch (mc)
    {
@@ -969,7 +1005,7 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
 void *MemoryManager::ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                                 size_t bytes, unsigned &flags)
 {
-   MemoryManager::CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
    if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
    MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
    if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -993,7 +1029,7 @@ void *MemoryManager::ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
 const void *MemoryManager::Read_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                                  size_t bytes, unsigned &flags)
 {
-   CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
    if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
    MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
    if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -1017,7 +1053,7 @@ const void *MemoryManager::Read_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
 void *MemoryManager::Write_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                             size_t bytes, unsigned &flags)
 {
-   CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
    if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
    MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
    if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -1061,22 +1097,20 @@ void MemoryManager::SyncAlias_(const void *base_h_ptr, void *alias_h_ptr,
                  (base_flags & (Mem::VALID_HOST | Mem::VALID_DEVICE));
 }
 
-MemoryType MemoryManager::GetDeviceMemoryType_(void *h_ptr)
+MemoryType MemoryManager::GetDeviceMemoryType_(void *h_ptr, bool alias)
 {
    if (mm.exists)
    {
-      const bool known = mm.IsKnown(h_ptr);
-      if (known)
+      if (!alias)
       {
-         internal::Memory &mem = maps->memories.at(h_ptr);
-         return mem.d_mt;
-      }
-      const bool alias = mm.IsAlias(h_ptr);
-      if (alias)
-      {
-         internal::Memory *mem = maps->aliases.at(h_ptr).mem;
-         return mem->d_mt;
+         auto iter = maps->memories.find(h_ptr);
+         MFEM_ASSERT(iter != maps->memories.end(), "internal error");
+         return iter->second.d_mt;
       }
+      // alias == true
+      auto iter = maps->aliases.find(h_ptr);
+      MFEM_ASSERT(iter != maps->aliases.end(), "internal error");
+      return iter->second.mem->d_mt;
    }
    MFEM_ABORT("internal error");
    return MemoryManager::host_mem_type;
@@ -1086,7 +1120,7 @@ MemoryType MemoryManager::GetHostMemoryType_(void *h_ptr)
 {
    if (!mm.exists) { return MemoryManager::host_mem_type; }
    if (mm.IsKnown(h_ptr)) { return maps->memories.at(h_ptr).h_mt; }
-   if (mm.IsAlias(h_ptr)) { return maps->aliases.at(h_ptr).mem->h_mt; }
+   if (mm.IsAlias(h_ptr)) { return maps->aliases.at(h_ptr).h_mt; }
    return MemoryManager::host_mem_type;
 }
 
@@ -1137,7 +1171,7 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
       {
          if (dst_h_ptr != src_d_ptr && bytes != 0)
          {
-            internal::Memory &src_d_base = maps->memories.at(src_d_ptr);
+            internal::Memory &src_d_base = maps->memories.at(src_h_ptr);
             MemoryType src_d_mt = src_d_base.d_mt;
             ctrl->Device(src_d_mt)->DtoH(dst_h_ptr, src_d_ptr, bytes);
          }
@@ -1240,6 +1274,10 @@ bool MemoryManager::IsAlias_(const void *h_ptr)
 void MemoryManager::Insert(void *h_ptr, size_t bytes,
                            MemoryType h_mt, MemoryType d_mt)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: registering h_ptr: " << h_ptr
+             << ", bytes: " << bytes << std::endl;
+#endif
    if (h_ptr == NULL)
    {
       MFEM_VERIFY(bytes == 0, "Trying to add NULL with size " << bytes);
@@ -1254,8 +1292,14 @@ void MemoryManager::Insert(void *h_ptr, size_t bytes,
    if (res.second == false)
    {
       auto &m = res.first->second;
-      MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt && m.d_mt == d_mt,
+      MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt &&
+                  (m.d_mt == d_mt || (d_mt == MemoryType::DEFAULT &&
+                                      m.d_mt == GetDualMemoryType(h_mt))),
                   "Address already present with different attributes!");
+#ifdef MFEM_TRACK_MEM_MANAGER
+      mfem::out << "[mfem memory manager]: repeated registration of h_ptr: "
+                << h_ptr << std::endl;
+#endif
    }
 #endif
 }
@@ -1267,7 +1311,7 @@ void MemoryManager::InsertDevice(void *d_ptr, void *h_ptr, size_t bytes,
    MFEM_ASSERT(h_ptr != NULL, "internal error");
    Insert(h_ptr, bytes, h_mt, d_mt);
    internal::Memory &mem = maps->memories.at(h_ptr);
-   if (d_ptr == NULL) { ctrl->Device(d_mt)->Alloc(mem); }
+   if (d_ptr == NULL && bytes != 0) { ctrl->Device(d_mt)->Alloc(mem); }
    else { mem.d_ptr = d_ptr; }
 }
 
@@ -1276,6 +1320,11 @@ void MemoryManager::InsertAlias(const void *base_ptr, void *alias_ptr,
 {
    size_t offset = static_cast<size_t>(static_cast<const char*>(alias_ptr) -
                                        static_cast<const char*>(base_ptr));
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: registering alias of base_ptr: "
+             << base_ptr << ", offset: " << offset << ", bytes: " << bytes
+             << ", base is alias: " << base_is_alias << std::endl;
+#endif
    if (!base_ptr)
    {
       MFEM_VERIFY(offset == 0,
@@ -1288,26 +1337,33 @@ void MemoryManager::InsertAlias(const void *base_ptr, void *alias_ptr,
       MFEM_ASSERT(alias.mem,"");
       base_ptr = alias.mem->h_ptr;
       offset += alias.offset;
+#ifdef MFEM_TRACK_MEM_MANAGER
+      mfem::out << "[mfem memory manager]: real base_ptr: " << base_ptr
+                << std::endl;
+#endif
    }
    internal::Memory &mem = maps->memories.at(base_ptr);
+   MFEM_VERIFY(offset + bytes <= mem.bytes, "invalid alias");
    auto res =
       maps->aliases.emplace(alias_ptr,
-                            internal::Alias{&mem, offset, bytes, 1, mem.h_mt});
+                            internal::Alias{&mem, offset, 1, mem.h_mt});
    if (res.second == false) // alias_ptr was already in the map
    {
-      if (res.first->second.mem != &mem || res.first->second.offset != offset)
-      {
-         mfem_error("alias already exists with different base/offset!");
-      }
-      else
-      {
-         res.first->second.counter++;
-      }
+      internal::Alias &alias = res.first->second;
+      // Update the alias data in case the existing alias is dangling
+      alias.mem = &mem;
+      alias.offset = offset;
+      alias.h_mt = mem.h_mt;
+      alias.counter++;
    }
 }
 
 void MemoryManager::Erase(void *h_ptr, bool free_dev_ptr)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: un-registering h_ptr: " << h_ptr
+             << std::endl;
+#endif
    if (!h_ptr) { return; }
    auto mem_map_iter = maps->memories.find(h_ptr);
    if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
@@ -1321,10 +1377,6 @@ void MemoryManager::EraseDevice(void *h_ptr)
    if (!h_ptr) { return; }
    auto mem_map_iter = maps->memories.find(h_ptr);
    if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
-   if (maps->aliases.find(h_ptr) != maps->aliases.end())
-   {
-      mfem_error("cannot delete aliased obj!");
-   }
    internal::Memory &mem = mem_map_iter->second;
    if (mem.d_ptr) { ctrl->Device(mem.d_mt)->Dealloc(mem);}
    mem.d_ptr = nullptr;
@@ -1332,6 +1384,10 @@ void MemoryManager::EraseDevice(void *h_ptr)
 
 void MemoryManager::EraseAlias(void *alias_ptr)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: un-registering alias_ptr: " << alias_ptr
+             << std::endl;
+#endif
    if (!alias_ptr) { return; }
    auto alias_map_iter = maps->aliases.find(alias_ptr);
    if (alias_map_iter == maps->aliases.end()) { mfem_error("Unknown alias!"); }
@@ -1355,14 +1411,14 @@ void *MemoryManager::GetDevicePtr(const void *h_ptr, size_t bytes,
    if (!mem.d_ptr)
    {
       if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
+      if (mem.bytes) { ctrl->Device(d_mt)->Alloc(mem); }
    }
    // Aliases might have done some protections
-   ctrl->Device(d_mt)->Unprotect(mem);
+   if (mem.d_ptr) { ctrl->Device(d_mt)->Unprotect(mem); }
    if (copy_data)
    {
       MFEM_ASSERT(bytes <= mem.bytes, "invalid copy size");
-      ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes);
+      if (bytes) { ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes); }
    }
    ctrl->Host(h_mt)->Protect(mem, bytes);
    return mem.d_ptr;
@@ -1388,16 +1444,17 @@ void *MemoryManager::GetAliasDevicePtr(const void *alias_ptr, size_t bytes,
    if (!mem.d_ptr)
    {
       if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
+      if (mem.bytes) { ctrl->Device(d_mt)->Alloc(mem); }
    }
    void *alias_h_ptr = static_cast<char*>(mem.h_ptr) + offset;
    void *alias_d_ptr = static_cast<char*>(mem.d_ptr) + offset;
    MFEM_ASSERT(alias_h_ptr == alias_ptr, "internal error");
-   MFEM_ASSERT(bytes <= alias.bytes, "internal error");
+   MFEM_ASSERT(offset + bytes <= mem.bytes, "internal error");
    mem.d_rw = mem.h_rw = false;
-   ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes);
+   if (mem.d_ptr) { ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes); }
    ctrl->Host(h_mt)->AliasUnprotect(alias_ptr, bytes);
-   if (copy) { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
+   if (copy && mem.d_ptr)
+   { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
    ctrl->Host(h_mt)->AliasProtect(alias_ptr, bytes);
    return alias_d_ptr;
 }
@@ -1504,6 +1561,23 @@ void MemoryManager::Configure(const MemoryType host_mt,
 void MemoryManager::Destroy()
 {
    MFEM_VERIFY(exists, "MemoryManager has already been destroyed!");
+#ifdef MFEM_TRACK_MEM_MANAGER
+   size_t num_memories = maps->memories.size();
+   size_t num_aliases = maps->aliases.size();
+   if (num_memories != 0 || num_aliases != 0)
+   {
+      MFEM_WARNING("...\n\t number of registered pointers: " << num_memories
+                   << "\n\t number of registered aliases : " << num_aliases);
+   }
+#endif
+   // Keep for debugging purposes:
+#if 0
+   mfem::out << "Destroying the MemoryManager ...\n"
+             << "remaining registered pointers : "
+             << maps->memories.size() << '\n'
+             << "remaining registered aliases  : "
+             << maps->aliases.size() << '\n';
+#endif
    for (auto& n : maps->memories)
    {
       internal::Memory &mem = n.second;
@@ -1554,7 +1628,6 @@ int MemoryManager::PrintAliases(std::ostream &out)
       out << "\nalias: key " << n.first << ", "
           << "h_ptr " << alias.mem->h_ptr << ", "
           << "offset " << alias.offset << ", "
-          << "bytes  " << alias.bytes << ", "
           << "counter " << alias.counter;
       n_out++;
    }
@@ -1569,7 +1642,13 @@ int MemoryManager::CompareHostAndDevice_(void *h_ptr, size_t size,
                  mm.GetAliasDevicePtr(h_ptr, size, false) :
                  mm.GetDevicePtr(h_ptr, size, false);
    char *h_buf = new char[size];
+#ifdef MFEM_USE_CUDA
    CuMemcpyDtoH(h_buf, d_ptr, size);
+#elif MFE_USE_HIP
+   HipMemcpyDtoH(h_buf, d_ptr, size);
+#else
+   std::memcpy(h_buf, d_ptr, size);
+#endif
    int res = std::memcmp(h_ptr, h_buf, size);
    delete [] h_buf;
    return res;
@@ -1591,13 +1670,24 @@ void MemoryPrintFlags(unsigned flags)
          << std::endl;
 }
 
-void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr)
+void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr,
+                                         bool alias)
 {
    if (!mm.exists) {return;}
-   const bool known = mm.IsKnown(h_ptr);
-   const bool alias = mm.IsAlias(h_ptr);
-   if (known) { MFEM_VERIFY(h_mt == maps->memories.at(h_ptr).h_mt,""); }
-   if (alias) { MFEM_VERIFY(h_mt == maps->aliases.at(h_ptr).mem->h_mt,""); }
+   if (!alias)
+   {
+      auto it = maps->memories.find(h_ptr);
+      MFEM_VERIFY(it != maps->memories.end(),
+                  "host pointer is not registered: h_ptr = " << h_ptr);
+      MFEM_VERIFY(h_mt == it->second.h_mt, "host pointer MemoryType mismatch");
+   }
+   else
+   {
+      auto it = maps->aliases.find(h_ptr);
+      MFEM_VERIFY(it != maps->aliases.end(),
+                  "alias pointer is not registered: h_ptr = " << h_ptr);
+      MFEM_VERIFY(h_mt == it->second.h_mt, "alias pointer MemoryType mismatch");
+   }
 }
 
 MemoryManager mm;

general/mem_manager.hpp

@@ -17,6 +17,9 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
+#ifdef MFEM_USE_MPI
+#include <HYPRE_config.h> // HYPRE_USING_CUDA
+#endif
 
 namespace mfem
 {
@@ -90,6 +93,9 @@ inline bool IsDeviceMemory(MemoryType mt)
 /// Return a suitable MemoryType for a given MemoryClass.
 MemoryType GetMemoryType(MemoryClass mc);
 
+/// Return true iff the MemoryType @a mt is contained in the MemoryClass @a mc.
+bool MemoryClassContainsType(MemoryClass mc, MemoryType mt);
+
 /// Return a suitable MemoryClass from a pair of MemoryClass%es.
 /** Note: this operation is commutative, i.e. a*b = b*a, associative, i.e.
     (a*b)*c = a*(b*c), and has an identity element: MemoryClass::HOST.
@@ -463,6 +469,13 @@ public:
        returned. */
    inline MemoryType GetMemoryType() const;
 
+   /// Return the host MemoryType of the Memory object.
+   inline MemoryType GetHostMemoryType() const { return h_mt; }
+
+   /** @brief Return the device MemoryType of the Memory object. If the device
+       MemoryType is not set, return MemoryType::DEFAULT. */
+   inline MemoryType GetDeviceMemoryType() const;
+
    /** @brief Return true if host pointer is valid */
    inline bool HostIsValid() const;
 
@@ -596,9 +609,9 @@ private: // Static methods used by the Memory<T> class
                           bool own, bool alias, unsigned &flags);
 
    /// Register a pair of external host and device pointers
-   static void Register_(void *h_ptr, void *d_ptr, size_t bytes,
-                         MemoryType h_mt, MemoryType d_mt,
-                         bool own, bool alias, unsigned &flags);
+   static void Register2_(void *h_ptr, void *d_ptr, size_t bytes,
+                          MemoryType h_mt, MemoryType d_mt,
+                          bool own, bool alias, unsigned &flags);
 
    /// Register an alias. Note: base_h_ptr may be an alias.
    static void Alias_(void *base_h_ptr, size_t offset, size_t bytes,
@@ -635,13 +648,13 @@ private: // Static methods used by the Memory<T> class
 
    /// Return the type the of the currently valid memory.
    /// If more than one types are valid, return a device type.
-   static MemoryType GetDeviceMemoryType_(void *h_ptr);
+   static MemoryType GetDeviceMemoryType_(void *h_ptr, bool alias);
 
    /// Return the type the of the host memory.
    static MemoryType GetHostMemoryType_(void *h_ptr);
 
    /// Verify that h_mt and h_ptr's h_mt (memory or alias) are equal.
-   static void CheckHostMemoryType_(MemoryType h_mt, void *h_ptr);
+   static void CheckHostMemoryType_(MemoryType h_mt, void *h_ptr, bool alias);
 
    /// Copy entries from valid memory type to valid memory type.
    ///  Both dest_h_ptr and src_h_ptr are registered host pointers.
@@ -851,15 +864,21 @@ inline void Memory<T>::Wrap(T *ptr, int size, bool own)
 {
    h_ptr = ptr;
    capacity = size;
-   const size_t bytes = size*sizeof(T);
    flags = (own ? OWNS_HOST : 0) | VALID_HOST;
    h_mt = MemoryManager::GetHostMemoryType();
 #ifdef MFEM_DEBUG
    if (own && MemoryManager::Exists())
-   { MFEM_VERIFY(h_mt == MemoryManager::GetHostMemoryType_(h_ptr),""); }
+   {
+      MemoryType h_ptr_mt = MemoryManager::GetHostMemoryType_(h_ptr);
+      MFEM_VERIFY(h_mt == h_ptr_mt,
+                  "h_mt = " << (int)h_mt << ", h_ptr_mt = " << (int)h_ptr_mt);
+   }
 #endif
    if (own && h_mt != MemoryType::HOST)
-   { MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags); }
+   {
+      const size_t bytes = size*sizeof(T);
+      MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags);
+   }
 }
 
 template <typename T>
@@ -880,7 +899,7 @@ inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
    else
    {
       h_mt = MemoryManager::GetDualMemoryType(mt);
-      h_ptr = (h_mt == MemoryType::HOST) ? new T[size] : nullptr;
+      h_ptr = (h_mt == MemoryType::HOST) ? NewHOST(size) : nullptr;
    }
    flags = 0;
    h_ptr = (T*)MemoryManager::Register_(ptr, h_ptr, size*sizeof(T), mt,
@@ -897,7 +916,8 @@ inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
    MFEM_ASSERT(IsHostMemory(h_mt),"");
    const size_t bytes = size*sizeof(T);
    const MemoryType d_mt = MemoryManager::GetDualMemoryType(h_mt);
-   MemoryManager::Register_(h_ptr, d_ptr, bytes, h_mt, d_mt, own, false, flags);
+   MemoryManager::Register2_(h_ptr, d_ptr, bytes, h_mt, d_mt,
+                             own, false, flags);
 }
 
 template <typename T>
@@ -907,13 +927,35 @@ inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
    h_mt = base.h_mt;
    h_ptr = base.h_ptr + offset;
    if (!(base.flags & REGISTERED))
-   { flags = (base.flags | ALIAS) & ~(OWNS_HOST | OWNS_DEVICE); }
-   else
    {
-      const size_t s_bytes = size*sizeof(T);
-      const size_t o_bytes = offset*sizeof(T);
-      MemoryManager::Alias_(base.h_ptr, o_bytes, s_bytes, base.flags, flags);
+      if (
+#ifndef HYPRE_USING_CUDA
+         // If the following condition is true then MemoryManager::Exists()
+         // should also be true:
+         IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
+#else
+         // When HYPRE_USING_CUDA is defined we always register the 'base' if
+         // the MemoryManager::Exists():
+         MemoryManager::Exists()
+#endif
+      )
+      {
+         // Register 'base':
+         MemoryManager::Register_(base.h_ptr, nullptr, base.capacity*sizeof(T),
+                                  base.h_mt, base.flags & OWNS_HOST,
+                                  base.flags & ALIAS, base.flags);
+      }
+      else
+      {
+         // Copy the flags from 'base', setting the ALIAS flag to true, and
+         // setting both OWNS_HOST and OWNS_DEVICE to false:
+         flags = (base.flags | ALIAS) & ~(OWNS_HOST | OWNS_DEVICE);
+         return;
+      }
    }
+   const size_t s_bytes = size*sizeof(T);
+   const size_t o_bytes = offset*sizeof(T);
+   MemoryManager::Alias_(base.h_ptr, o_bytes, s_bytes, base.flags, flags);
 }
 
 template <typename T>
@@ -1069,7 +1111,14 @@ template <typename T>
 inline MemoryType Memory<T>::GetMemoryType() const
 {
    if (!(flags & VALID_DEVICE)) { return h_mt; }
-   return MemoryManager::GetDeviceMemoryType_(h_ptr);
+   return MemoryManager::GetDeviceMemoryType_(h_ptr, flags & ALIAS);
+}
+
+template <typename T>
+inline MemoryType Memory<T>::GetDeviceMemoryType() const
+{
+   if (!(flags & REGISTERED)) { return MemoryType::DEFAULT; }
+   return MemoryManager::GetDeviceMemoryType_(h_ptr, flags & ALIAS);
 }
 
 template <typename T>

general/version.cpp

@@ -82,6 +82,9 @@ const char *GetConfigStr()
 #ifdef MFEM_USE_CUDA
       "MFEM_USE_CUDA\n"
 #endif
+#ifdef MFEM_USE_EPIC
+      "MFEM_USE_EPIC\n"
+#endif
 #ifdef MFEM_USE_EXCEPTIONS
       "MFEM_USE_EXCEPTIONS\n"
 #endif

linalg/CMakeLists.txt

@@ -83,6 +83,11 @@ if (MFEM_USE_SUNDIALS)
   list(APPEND HDRS sundials.hpp)
 endif()
 
+if (MFEM_USE_EPIC)
+  list(APPEND SRCS epic.cpp)
+  list(APPEND HDRS epic.hpp)
+endif()
+
 if (MFEM_USE_SUPERLU)
   list(APPEND SRCS superlu.cpp)
   # If this list (HDRS -> HEADERS) is used for install, we probably want the

linalg/blockoperator.cpp

@@ -97,11 +97,6 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
    {
       yblock.GetBlock(iRow).SyncAliasMemory(y);
    }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }
 
 // Action of the transpose operator
@@ -133,11 +128,6 @@ void BlockOperator::MultTranspose (const Vector & x, Vector & y) const
    {
       yblock.GetBlock(iRow).SyncAliasMemory(y);
    }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }
 
 BlockOperator::~BlockOperator()
@@ -208,11 +198,6 @@ void BlockDiagonalPreconditioner::Mult (const Vector & x, Vector & y) const
    {
       yblock.GetBlock(i).SyncAliasMemory(y);
    }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }
 
 // Action of the transpose operator
@@ -244,11 +229,6 @@ void BlockDiagonalPreconditioner::MultTranspose (const Vector & x,
    {
       yblock.GetBlock(i).SyncAliasMemory(y);
    }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }
 
 BlockDiagonalPreconditioner::~BlockDiagonalPreconditioner()

linalg/blockoperator.hpp

@@ -51,6 +51,12 @@ public:
     */
    BlockOperator(const Array<int> & row_offsets, const Array<int> & col_offsets);
 
+   /// Copy assignment is not supported
+   BlockOperator &operator=(const BlockOperator &) = delete;
+
+   /// Move assignment is not supported
+   BlockOperator &operator=(BlockOperator &&) = delete;
+
    //! Add block op in the block-entry (iblock, iblock).
    /**
     * iblock: The block will be inserted in location (iblock, iblock).

linalg/blockvector.cpp

@@ -20,9 +20,7 @@ void BlockVector::SetBlocks()
 {
    for (int i = 0; i < numBlocks; ++i)
    {
-      blocks[i].NewMemoryAndSize(
-         Memory<double>(data, blockOffsets[i], BlockSize(i)),
-         BlockSize(i), true);
+      blocks[i].MakeRef(*this, blockOffsets[i], BlockSize(i));
    }
 }
 
@@ -74,6 +72,16 @@ BlockVector::BlockVector(double *data, const Array<int> & bOffsets):
    SetBlocks();
 }
 
+BlockVector::BlockVector(Vector &v, const Array<int> &bOffsets)
+   : Vector(),
+     numBlocks(bOffsets.Size()-1),
+     blockOffsets(bOffsets.GetData())
+{
+   MakeRef(v, 0, blockOffsets[numBlocks]);
+   blocks = new Vector[numBlocks];
+   SetBlocks();
+}
+
 void BlockVector::Update(double *data, const Array<int> & bOffsets)
 {
    NewDataAndSize(data, bOffsets.Last());
@@ -178,9 +186,23 @@ BlockVector::~BlockVector()
 
 void BlockVector::GetBlockView(int i, Vector & blockView)
 {
-   blockView.NewMemoryAndSize(
-      Memory<double>(data, blockOffsets[i], BlockSize(i)),
-      BlockSize(i), true);
+   blockView.MakeRef(*this, blockOffsets[i], BlockSize(i));
+}
+
+void BlockVector::SyncToBlocks() const
+{
+   for (int i = 0; i < numBlocks; ++i)
+   {
+      blocks[i].SyncMemory(*this);
+   }
+}
+
+void BlockVector::SyncFromBlocks() const
+{
+   for (int i = 0; i < numBlocks; ++i)
+   {
+      blocks[i].SyncAliasMemory(*this);
+   }
 }
 
 }

linalg/blockvector.hpp

@@ -72,6 +72,9 @@ public:
     */
    BlockVector(double *data, const Array<int> & bOffsets);
 
+   /// Wrap a Vector as a BlockVector
+   BlockVector(Vector &v, const Array<int> &bOffsets);
+
    //! Return the number of blocks
    int NumBlocks() const { return numBlocks; }
 
@@ -117,6 +120,21 @@ public:
        - currently, the block-vector does not own its data, or
        - currently, the block-vector does not use MemoryType @a mt. */
    void Update(const Array<int> &bOffsets, MemoryType mt);
+
+   /** @brief Synchronize the memory location flags (i.e. the memory validity
+       flags) of the big/monolithic block-vector with its sub-vector blocks. The
+       big/monolithic vector has the correct memory location flags. */
+   /** This method will copy the data validity flags from the big/monolithic
+       block-vector to its sub-vector block. */
+   void SyncToBlocks() const;
+
+   /** @brief Synchronize the memory location flags (i.e. the memory validity
+       flags) of the big/monolithic block-vector with its sub-vector blocks. The
+       sub-vector blocks have the correct memory location flags. */
+   /** This method will copy/move the data of the sub-vector blocks (if
+       necessary) so that each block matches the memory location flags of the
+       big/monolithic block-vector. */
+   void SyncFromBlocks() const;
 };
 
 }

linalg/complex_operator.cpp

@@ -81,13 +81,6 @@ void ComplexOperator::Mult(const Vector &x, Vector &y) const
 
    y_r_.SyncAliasMemory(y);
    y_i_.SyncAliasMemory(y);
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   x_r_.Destroy();
-   x_i_.Destroy();
-   y_r_.Destroy();
-   y_i_.Destroy();
 }
 
 void ComplexOperator::Mult(const Vector &x_r, const Vector &x_i,
@@ -137,13 +130,6 @@ void ComplexOperator::MultTranspose(const Vector &x, Vector &y) const
 
    y_r_.SyncAliasMemory(y);
    y_i_.SyncAliasMemory(y);
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   x_r_.Destroy();
-   x_i_.Destroy();
-   y_r_.Destroy();
-   y_i_.Destroy();
 }
 
 void ComplexOperator::MultTranspose(const Vector &x_r, const Vector &x_i,
@@ -584,10 +570,10 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const
                                            global_num_cols_i);
 
    int row_starts_size = (HYPRE_AssumedPartitionCheck()) ? 2 : nranks_ + 1;
-   HYPRE_BigInt * row_starts = mfem_hypre_CTAlloc(HYPRE_BigInt,
-                                                  row_starts_size);
-   HYPRE_BigInt * col_starts = mfem_hypre_CTAlloc(HYPRE_BigInt,
-                                                  row_starts_size);
+   HYPRE_BigInt * row_starts = mfem_hypre_CTAlloc_host(HYPRE_BigInt,
+                                                       row_starts_size);
+   HYPRE_BigInt * col_starts = mfem_hypre_CTAlloc_host(HYPRE_BigInt,
+                                                       row_starts_size);
 
    const HYPRE_BigInt * row_starts_z = (A_r) ? A_r->RowPart() :
                                        ((A_i) ? A_i->RowPart() : NULL);
@@ -665,14 +651,15 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const
    int offd_nnz = 2 * (offd_r_nnz + offd_i_nnz);
 
    // Allocate CSR arrays for the combined matrix
-   HYPRE_Int * diag_I = mfem_hypre_CTAlloc(HYPRE_Int, 2 * nrows + 1);
-   HYPRE_Int * diag_J = mfem_hypre_CTAlloc(HYPRE_Int, diag_nnz);
-   double    * diag_D = mfem_hypre_CTAlloc(double, diag_nnz);
+   HYPRE_Int * diag_I = mfem_hypre_CTAlloc_host(HYPRE_Int, 2 * nrows + 1);
+   HYPRE_Int * diag_J = mfem_hypre_CTAlloc_host(HYPRE_Int, diag_nnz);
+   double    * diag_D = mfem_hypre_CTAlloc_host(double, diag_nnz);
 
-   HYPRE_Int * offd_I = mfem_hypre_CTAlloc(HYPRE_Int, 2 * nrows + 1);
-   HYPRE_Int * offd_J = mfem_hypre_CTAlloc(HYPRE_Int, offd_nnz);
-   double    * offd_D = mfem_hypre_CTAlloc(double, offd_nnz);
-   HYPRE_BigInt * cmap   = mfem_hypre_CTAlloc(HYPRE_BigInt, 2 * num_cols_offd);
+   HYPRE_Int * offd_I = mfem_hypre_CTAlloc_host(HYPRE_Int, 2 * nrows + 1);
+   HYPRE_Int * offd_J = mfem_hypre_CTAlloc_host(HYPRE_Int, offd_nnz);
+   double    * offd_D = mfem_hypre_CTAlloc_host(double, offd_nnz);
+   HYPRE_BigInt * cmap = mfem_hypre_CTAlloc_host(HYPRE_BigInt,
+                                                 2 * num_cols_offd);
 
    // Fill the CSR arrays for the diagonal portion of the matrix
    const double factor = (convention_ == HERMITIAN) ? 1.0 : -1.0;
@@ -797,14 +784,13 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const
                                            row_starts, col_starts,
                                            diag_I, diag_J, diag_D,
                                            offd_I, offd_J, offd_D,
-                                           2 * num_cols_offd, cmap);
+                                           2 * num_cols_offd, cmap,
+                                           true);
 
-   // Give the new matrix ownership of its internal arrays
-   A->SetOwnerFlags(-1,-1,-1);
-   hypre_CSRMatrixSetDataOwner(((hypre_ParCSRMatrix*)(*A))->diag,1);
-   hypre_CSRMatrixSetDataOwner(((hypre_ParCSRMatrix*)(*A))->offd,1);
-   hypre_ParCSRMatrixSetRowStartsOwner((hypre_ParCSRMatrix*)(*A),1);
-   hypre_ParCSRMatrixSetColStartsOwner((hypre_ParCSRMatrix*)(*A),1);
+   // Give the new matrix ownership of row_starts and col_starts
+   hypre_ParCSRMatrix *hA = (hypre_ParCSRMatrix*)(*A);
+   hypre_ParCSRMatrixSetRowStartsOwner(hA,1);
+   hypre_ParCSRMatrixSetColStartsOwner(hA,1);
 
    return A;
 }

linalg/epic.cpp

@@ -0,0 +1,171 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "epic.hpp"
+
+#ifdef MFEM_USE_EPIC
+
+namespace mfem
+{
+
+EPICSolver::EPICSolver(bool exactJacobian_, EPICNumJacDelta delta)
+{
+   // Allocate an empty serial N_Vector
+   temp = N_VNewEmpty_Serial(0);
+   m[0] = 10;
+   m[1] = 10;
+   MFEM_VERIFY(temp, "error in N_VNewEmpty_Serial()");
+   exactJacobian = exactJacobian_;
+   Jtv = NULL;
+   Delta = delta;
+}
+
+#ifdef MFEM_USE_MPI
+EPICSolver::EPICSolver(MPI_Comm comm)
+{
+   m[0] = 10;
+   m[1] = 10;
+
+   // Allocate an empty vector
+   if (comm == MPI_COMM_NULL)
+   {
+      // Allocate an empty serial N_Vector
+      temp = N_VNewEmpty_Serial(0);
+      MFEM_VERIFY(temp, "error in N_VNewEmpty_Serial()");
+   }
+   else
+   {
+      // Allocate an empty parallel N_Vector
+      temp = N_VNewEmpty_Parallel(comm, 0, 0);  // calls MPI_Allreduce()
+      MFEM_VERIFY(temp, "error in N_VNewEmpty_Parallel()");
+   }
+}
+#endif
+
+int EPICSolver::RHS(realtype t, const N_Vector y, N_Vector ydot, void *user_data)
+{
+   // Get data from N_Vectors
+   const Vector mfem_y(y);
+   Vector mfem_ydot(ydot);
+   EPICSolver *self = static_cast<EPICSolver*>(user_data);
+
+   // Compute y' = f(t, y)
+   self->f->SetTime(t);
+   self->f->Mult(mfem_y, mfem_ydot);
+
+   // Return success
+   return 0;
+}
+
+
+int EPICSolver::Jacobian(N_Vector v, N_Vector Jv, realtype t, N_Vector y, N_Vector fy, void *user_data, N_Vector tmp)
+{
+    // Get data from N_Vectors
+   const Vector mfem_v(v);
+   Vector mfem_Jv(Jv);
+   EPICSolver *self = static_cast<EPICSolver*>(user_data);
+
+   // Compute J(t, y) v
+   self->Jtv->Mult(mfem_v, mfem_Jv);
+
+   return 0;
+}
+
+void EPICSolver::Init(TimeDependentOperator &f)
+{
+    ODESolver::Init(f);
+	    
+    long local_size = f.Height();
+    long global_size = 0;
+#ifdef MFEM_USE_MPI
+    if (Parallel())
+    {
+        MPI_Allreduce(&local_size, &global_size, 1, MPI_LONG, MPI_SUM,
+                      NV_COMM_P(temp));
+    }
+#endif
+
+    Vector mfem_temp(local_size);
+    mfem_temp.ToNVector(temp, global_size);
+}
+
+EPI2::EPI2(bool exactJacobian, EPICNumJacDelta delta) : EPICSolver(exactJacobian, delta) {}
+
+void EPI2::Init(TimeDependentOperator &f)
+{
+    EPICSolver::Init(f);
+    long local_size = f.Height();
+    if (exactJacobian) {
+       integrator = new Epi2_KIOPS(EPICSolver::RHS, EPICSolver::Jacobian, this, 100, temp ,local_size);
+    } else {
+       integrator = new Epi2_KIOPS(EPICSolver::RHS, Delta, this, 100, temp ,local_size);
+    }
+
+}
+
+EPIRK4::EPIRK4(bool exactJacobian, EPICNumJacDelta delta) : EPICSolver(exactJacobian, delta) {}
+
+void EPIRK4::Init(TimeDependentOperator &f)
+{
+    EPICSolver::Init(f);
+    long local_size = f.Height();
+    if (exactJacobian) {
+       integrator = new EpiRK4SC_KIOPS(EPICSolver::RHS, EPICSolver::Jacobian, this, 100, temp ,local_size);
+    } else {
+       integrator = new EpiRK4SC_KIOPS(EPICSolver::RHS, Delta, this, 100, temp ,local_size);
+    }
+}
+
+void EPICSolver::Step(Vector &x, double &t, double &dt)
+{
+   if (!Parallel())
+   {
+      NV_DATA_S(temp) = x.GetData();
+      MFEM_VERIFY(NV_LENGTH_S(temp) == x.Size(), "");
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      NV_DATA_P(temp) = x.GetData();
+      MFEM_VERIFY(NV_LOCLENGTH_P(temp) == x.Size(), "");
+#endif
+   }
+
+   Jtv = &(this->f->GetGradient(x));
+}
+
+void EPI2::Step(Vector &x, double &t, double &dt)
+{
+    EPICSolver::Step(x, t, dt);
+    integrator->Integrate(dt, t, t+dt, 0, temp, 1e-10, m);
+    t += dt;
+}
+
+void EPIRK4::Step(Vector &x, double &t, double &dt)
+{
+    EPICSolver::Step(x, t, dt);
+    integrator->Integrate(dt, t, t+dt, 0, temp, 1e-10, m);
+    t += dt;
+}
+
+EPI2::~EPI2()
+{
+    delete integrator;
+}
+
+EPIRK4::~EPIRK4()
+{
+    delete integrator;
+}
+
+}
+
+#endif

linalg/epic.hpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_EPIC
+#define MFEM_EPIC
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_EPIC
+
+// SUNDIALS vectors
+#include <nvector/nvector_serial.h>
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#include <nvector/nvector_parallel.h>
+#endif
+
+#include "ode.hpp"
+#include "solvers.hpp"
+#include <Epic.h>
+
+namespace mfem
+{
+
+typedef void (*JacobianFun)(const realtype t, const Vector &y, const Vector& v, Vector& Jv, void* user_data);
+// ---------------------------------------------------------------------------
+// Interface to the EPIC library -- exponential methods
+// ---------------------------------------------------------------------------
+
+class EPICSolver : public ODESolver
+{
+protected:
+    EPICNumJacDelta Delta;
+    Operator* Jtv;
+    N_Vector temp;
+    int m[2];
+
+    bool exactJacobian;
+
+    #ifdef MFEM_USE_MPI
+    bool Parallel() const
+    {
+        return (N_VGetVectorID(temp) != SUNDIALS_NVEC_SERIAL);
+    }
+    #else
+    bool Parallel() const { return false; }
+    #endif
+
+public:
+    EPICSolver(bool exactJacobian, EPICNumJacDelta delta=&DefaultDelta);
+    EPICSolver(MPI_Comm comm);
+
+    static int RHS(realtype t, const N_Vector y, N_Vector ydot, void *user_data);
+    static int Jacobian(N_Vector v, N_Vector Jv, realtype t,
+                         N_Vector y, N_Vector fy, void *user_data, N_Vector tmp);
+    virtual void Init(TimeDependentOperator &f);
+    virtual void Step(Vector &x, double &t, double &dt);
+
+    virtual ~EPICSolver() {}
+};
+
+class EPI2 : public EPICSolver
+{
+protected:
+    Epi2_KIOPS* integrator;
+public:
+    EPI2(bool exactJacobian=true, EPICNumJacDelta delta=&DefaultDelta);
+    virtual void Init(TimeDependentOperator &f);
+    virtual void Step(Vector &x, double &t, double &dt);
+
+    virtual ~EPI2();
+};
+
+class EPIRK4 : public EPICSolver
+{
+protected:
+    EpiRK4SC_KIOPS* integrator;
+public:
+    EPIRK4(bool exactJacobian=true, EPICNumJacDelta delta=&DefaultDelta);
+    virtual void Init(TimeDependentOperator &f);
+    virtual void Step(Vector &x, double &t, double &dt);
+
+    virtual ~EPIRK4();
+};
+
+} // namespace mfem
+
+#endif // MFEM_USE_EPIC
+
+#endif // MFEM_EPIC

linalg/hypre.hpp

@@ -32,6 +32,10 @@
 #error "MFEM does not work with HYPRE's complex numbers support"
 #endif
 
+#if defined(HYPRE_USING_CUDA) && !defined(MFEM_USE_CUDA)
+#error "MFEM_USE_CUDA=YES is required when HYPRE is built with CUDA!"
+#endif
+
 #include "sparsemat.hpp"
 #include "hypre_parcsr.hpp"
 
@@ -64,6 +68,31 @@ inline int to_int(HYPRE_Int i)
 }
 #endif
 
+} // namespace internal
+
+
+/// The MemoryClass used by Hypre objects.
+inline constexpr MemoryClass GetHypreMemoryClass()
+{
+#ifndef HYPRE_USING_CUDA
+   return MemoryClass::HOST;
+#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryClass::MANAGED;
+#else
+   return MemoryClass::DEVICE;
+#endif
+}
+
+/// The MemoryType used by MFEM when allocating arrays for Hypre objects.
+inline MemoryType GetHypreMemoryType()
+{
+#ifndef HYPRE_USING_CUDA
+   return Device::GetHostMemoryType();
+#elif defined(HYPRE_USING_UNIFIED_MEMORY)
+   return MemoryType::MANAGED;
+#else
+   return MemoryType::DEVICE;
+#endif
 }
 
 /// Wrapper for hypre's parallel vector class
@@ -106,9 +135,12 @@ public:
        columns, and data. */
    /** The data must be allocated and destroyed outside. If @a data_ is NULL, a
        dummy vector without a valid data array will be created. See @ref
-       hypre_partitioning_descr "here" for a description of the @a col array. */
+       hypre_partitioning_descr "here" for a description of the @a col array.
+
+       If @a is_device_ptr is true, the pointer @a data_ is assumed to be
+       allocated in the memory location HYPRE_MEMORY_DEVICE. */
    HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size, double *data_,
-                  HYPRE_BigInt *col);
+                  HYPRE_BigInt *col, bool is_device_ptr = false);
    /// Creates vector compatible with y
    HypreParVector(const HypreParVector &y);
    /// Creates vector compatible with (i.e. in the domain of) A or A^T
@@ -144,7 +176,7 @@ public:
    /// Typecasting to hypre's HYPRE_ParVector, a.k.a. void *
    operator HYPRE_ParVector() const { return (HYPRE_ParVector) x; }
 #endif
-   /// Changes the ownership of the the vector
+   /// Changes the ownership of the vector
    hypre_ParVector *StealParVector() { own_ParVector = 0; return x; }
 
    /// Sets ownership of the internal hypre_ParVector
@@ -161,12 +193,59 @@ public:
    /// Define '=' for hypre vectors.
    HypreParVector& operator= (const HypreParVector &y);
 
+   using Vector::Read;
+
    /// Sets the data of the Vector and the hypre_ParVector to @a data_.
    /** Must be used only for HypreParVector%s that do not own the data,
        e.g. created with the constructor:
        HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *). */
    void SetData(double *data_);
 
+   /** @brief Prepare the HypreParVector for read access in hypre's device
+       memory space, HYPRE_MEMORY_DEVICE. */
+   void HypreRead() const;
+
+   /** @brief Prepare the HypreParVector for read and write access in hypre's
+       device memory space, HYPRE_MEMORY_DEVICE. */
+   void HypreReadWrite();
+
+   /** @brief Prepare the HypreParVector for write access in hypre's device
+       memory space, HYPRE_MEMORY_DEVICE. */
+   void HypreWrite();
+
+   /** @brief Replace the HypreParVector's data with the given Memory, @a mem,
+       and prepare the vector for read access in hypre's device memory space,
+       HYPRE_MEMORY_DEVICE. */
+   /** This method must be used with HypreParVector%s that do not own the data,
+       e.g. created with the constructor:
+       HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *).
+
+       The Memory @a mem must be accessible with the hypre MemoryClass defined
+       by GetHypreMemoryClass(). */
+   void WrapMemoryRead(const Memory<double> &mem);
+
+   /** @brief Replace the HypreParVector's data with the given Memory, @a mem,
+       and prepare the vector for read and write access in hypre's device memory
+       space, HYPRE_MEMORY_DEVICE. */
+   /** This method must be used with HypreParVector%s that do not own the data,
+       e.g. created with the constructor:
+       HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *).
+
+       The Memory @a mem must be accessible with the hypre MemoryClass defined
+       by GetHypreMemoryClass(). */
+   void WrapMemoryReadWrite(Memory<double> &mem);
+
+   /** @brief Replace the HypreParVector's data with the given Memory, @a mem,
+       and prepare the vector for write access in hypre's device memory space,
+       HYPRE_MEMORY_DEVICE. */
+   /** This method must be used with HypreParVector%s that do not own the data,
+       e.g. created with the constructor:
+       HypreParVector(MPI_Comm, HYPRE_BigInt, double *, HYPRE_BigInt *).
+
+       The Memory @a mem must be accessible with the hypre MemoryClass defined
+       by GetHypreMemoryClass(). */
+   void WrapMemoryWrite(Memory<double> &mem);
+
    /// Set random values
    HYPRE_Int Randomize(HYPRE_Int seed);
 
@@ -203,15 +282,23 @@ private:
 
    /// Auxiliary vectors for typecasting
    mutable HypreParVector *X, *Y;
+   /** @brief Auxiliary buffers for the case when the input or output arrays in
+       methods like Mult(double, const Vector &, double, Vector &) need to be
+       deep copied in order to be used by hypre. */
+   mutable Memory<double> auxX, auxY;
 
    // Flags indicating ownership of A->diag->{i,j,data}, A->offd->{i,j,data},
    // and A->col_map_offd.
    // The possible values for diagOwner are:
    //  -1: no special treatment of A->diag (default)
+   //      when hypre is built with CUDA support, A->diag owns the "host"
+   //      pointers (according to A->diag->owns_data)
+   //  -2: used when hypre is built with CUDA support, A->diag owns the "hypre"
+   //      pointers (according to A->diag->owns_data)
    //   0: prevent hypre from destroying A->diag->{i,j,data}
-   //   1: same as 0, plus take ownership of A->diag->{i,j}
-   //   2: same as 0, plus take ownership of A->diag->data
-   //   3: same as 0, plus take ownership of A->diag->{i,j,data}
+   //   1: same as 0, plus own the "host" A->diag->{i,j}
+   //   2: same as 0, plus own the "host" A->diag->data
+   //   3: same as 0, plus own the "host" A->diag->{i,j,data}
    // The same values and rules apply to offdOwner and A->offd.
    // The possible values for colMapOwner are:
    //  -1: no special treatment of A->col_map_offd (default)
@@ -223,39 +310,54 @@ private:
    // Does the object own the pointer A?
    signed char ParCSROwner;
 
+   MemoryIJData mem_diag, mem_offd;
+
    // Initialize with defaults. Does not initialize inherited members.
    void Init();
 
    // Delete all owned data. Does not perform re-initialization with defaults.
    void Destroy();
 
+   void Read(MemoryClass mc) const;
+   void ReadWrite(MemoryClass mc);
+   // The Boolean flags are used in Destroy().
+   void Write(MemoryClass mc, bool set_diag = true, bool set_offd = true);
+
    // Copy (shallow/deep, based on HYPRE_BIGINT) the I and J arrays from csr to
    // hypre_csr. Shallow copy the data. Return the appropriate ownership flag.
-   static char CopyCSR(SparseMatrix *csr, hypre_CSRMatrix *hypre_csr);
+   // The CSR arrays are wrapped in the mem_csr struct which is used to move
+   // these arrays to device, if necessary.
+   static signed char CopyCSR(SparseMatrix *csr,
+                              MemoryIJData &mem_csr,
+                              hypre_CSRMatrix *hypre_csr,
+                              bool mem_owner);
    // Copy (shallow or deep, based on HYPRE_BIGINT) the I and J arrays from
    // bool_csr to hypre_csr. Allocate the data array and set it to all ones.
-   // Return the appropriate ownership flag.
-   static char CopyBoolCSR(Table *bool_csr, hypre_CSRMatrix *hypre_csr);
+   // Return the appropriate ownership flag. The CSR arrays are wrapped in the
+   // mem_csr struct which is used to move these arrays to device, if necessary.
+   static signed char CopyBoolCSR(Table *bool_csr,
+                                  MemoryIJData &mem_csr,
+                                  hypre_CSRMatrix *hypre_csr);
 
    // Copy the j array of a hypre_CSRMatrix to the given J array, converting
    // the indices from HYPRE_Int/HYPRE_BigInt to int.
    static void CopyCSR_J(hypre_CSRMatrix *hypre_csr, int *J);
 
+   // Wrap the data from h_mat into mem with the given ownership flag.
+   // If the new Memory arrays in mem are not suitable to be accessed via
+   // GetHypreMemoryClass(), then mem will be re-allocated using the memory type
+   // returned by GetHypreMemoryType(), the data will be deep copied, and h_mat
+   // will be updated with the new pointers.
+   static signed char HypreCsrToMem(hypre_CSRMatrix *h_mat, MemoryType h_mat_mt,
+                                    bool own_ija, MemoryIJData &mem);
+
 public:
    /// An empty matrix to be used as a reference to an existing matrix
    HypreParMatrix();
 
    /// Converts hypre's format to HypreParMatrix
    /** If @a owner is false, ownership of @a a is not transferred */
-   void WrapHypreParCSRMatrix(hypre_ParCSRMatrix *a, bool owner = true)
-   {
-      Destroy();
-      Init();
-      A = a;
-      ParCSROwner = owner;
-      height = GetNumRows();
-      width = GetNumCols();
-   }
+   void WrapHypreParCSRMatrix(hypre_ParCSRMatrix *a, bool owner = true);
 
    /// Converts hypre's format to HypreParMatrix
    /** If @a owner is false, ownership of @a a is not transferred */
@@ -274,7 +376,8 @@ public:
        @warning The ordering of the columns in each row in @a *diag may be
        changed by this constructor to ensure that the first entry in each row is
        the diagonal one. This is expected by most hypre functions. */
-   HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size, HYPRE_BigInt *row_starts,
+   HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size,
+                  HYPRE_BigInt *row_starts,
                   SparseMatrix *diag); // constructor with 4 arguments, v1
 
    /// Creates block-diagonal rectangular parallel matrix.
@@ -289,25 +392,36 @@ public:
 
    /// Creates general (rectangular) parallel matrix.
    /** The new HypreParMatrix does not take ownership of any of the input
-       arrays. See @ref hypre_partitioning_descr "here" for a description of the
+       arrays, if @a own_diag_offd is false (default). If @a own_diag_offd is
+       true, ownership of @a diag and @a offd is transferred to the
+       HypreParMatrix.
+
+       See @ref hypre_partitioning_descr "here" for a description of the
        partitioning arrays @a row_starts and @a col_starts. */
    HypreParMatrix(MPI_Comm comm, HYPRE_BigInt global_num_rows,
                   HYPRE_BigInt global_num_cols, HYPRE_BigInt *row_starts,
-                  HYPRE_BigInt *col_starts, SparseMatrix *diag, SparseMatrix *offd,
-                  HYPRE_BigInt *cmap); // constructor with 8 arguments
+                  HYPRE_BigInt *col_starts, SparseMatrix *diag,
+                  SparseMatrix *offd, HYPRE_BigInt *cmap,
+                  bool own_diag_offd = false); // constructor with 8+1 arguments
 
    /// Creates general (rectangular) parallel matrix.
    /** The new HypreParMatrix takes ownership of all input arrays, except
        @a col_starts and @a row_starts. See @ref hypre_partitioning_descr "here"
        for a description of the partitioning arrays @a row_starts and @a
-       col_starts. */
+       col_starts.
+
+       If @a hypre_arrays is false, all arrays (except @a row_starts and
+       @a col_starts) are assumed to be allocated according to the MemoryType
+       returned by Device::GetHostMemoryType(). If @a hypre_arrays is true, then
+       the same arrays are assumed to be allocated by hypre as host arrays. */
    HypreParMatrix(MPI_Comm comm,
                   HYPRE_BigInt global_num_rows, HYPRE_BigInt global_num_cols,
                   HYPRE_BigInt *row_starts, HYPRE_BigInt *col_starts,
                   HYPRE_Int *diag_i, HYPRE_Int *diag_j, double *diag_data,
                   HYPRE_Int *offd_i, HYPRE_Int *offd_j, double *offd_data,
                   HYPRE_Int offd_num_cols,
-                  HYPRE_BigInt *offd_col_map); // constructor with 13 arguments
+                  HYPRE_BigInt *offd_col_map,
+                  bool hypre_arrays = false); // constructor with 13+1 arguments
 
    /// Creates a parallel matrix from SparseMatrix on processor 0.
    /** See @ref hypre_partitioning_descr "here" for a description of the
@@ -363,12 +477,11 @@ public:
    /// Typecasting to hypre's HYPRE_ParCSRMatrix, a.k.a. void *
    operator HYPRE_ParCSRMatrix() { return (HYPRE_ParCSRMatrix) A; }
 #endif
-   /// Changes the ownership of the the matrix
+   /// Changes the ownership of the matrix
    hypre_ParCSRMatrix* StealData();
 
    /// Explicitly set the three ownership flags, see docs for diagOwner etc.
-   void SetOwnerFlags(signed char diag, signed char offd, signed char colmap)
-   { diagOwner = diag, offdOwner = offd, colMapOwner = colmap; }
+   void SetOwnerFlags(signed char diag, signed char offd, signed char colmap);
 
    /// Get diag ownership flag
    signed char OwnsDiag() const { return diagOwner; }
@@ -439,7 +552,7 @@ public:
        with relative size > @a threshold in *this. */
 #if MFEM_HYPRE_VERSION >= 21800
    HypreParMatrix *ExtractSubmatrix(const Array<int> &indices,
-                                    double threshhold=0.0) const;
+                                    double threshold=0.0) const;
 #endif
 
    /// Returns the number of rows in the diagonal block of the ParCSRMatrix
@@ -474,6 +587,8 @@ public:
        partitioning array. */
    HYPRE_BigInt *GetColStarts() const { return hypre_ParCSRMatrixColStarts(A); }
 
+   virtual MemoryClass GetMemoryClass() const { return GetHypreMemoryClass(); }
+
    /// Computes y = alpha * A * x + beta * y
    HYPRE_Int Mult(HypreParVector &x, HypreParVector &y,
                   double alpha = 1.0, double beta = 0.0) const;
@@ -492,26 +607,32 @@ public:
    virtual void MultTranspose(const Vector &x, Vector &y) const
    { MultTranspose(1.0, x, 0.0, y); }
 
-   /// Computes y = a * |A| * x + b * y, using entry-wise absolute values of matrix A
+   /** @brief Computes y = a * |A| * x + b * y, using entry-wise absolute values
+       of the matrix A. */
    void AbsMult(double a, const Vector &x, double b, Vector &y) const;
 
-   /// Computes y = a * |At| * x + b * y, using entry-wise absolute values of the transpose of matrix A
+   /** @brief Computes y = a * |At| * x + b * y, using entry-wise absolute
+       values of the transpose of the matrix A. */
    void AbsMultTranspose(double a, const Vector &x, double b, Vector &y) const;
 
-   /** The "Boolean" analog of y = alpha * A * x + beta * y, where elements in
-       the sparsity pattern of the matrix are treated as "true". */
+   /** @brief The "Boolean" analog of y = alpha * A * x + beta * y, where
+       elements in the sparsity pattern of the matrix are treated as "true". */
    void BooleanMult(int alpha, const int *x, int beta, int *y)
    {
+      HostRead();
       internal::hypre_ParCSRMatrixBooleanMatvec(A, alpha, const_cast<int*>(x),
                                                 beta, y);
+      HypreRead();
    }
 
-   /** The "Boolean" analog of y = alpha * A^T * x + beta * y, where elements in
-       the sparsity pattern of the matrix are treated as "true". */
+   /** @brief The "Boolean" analog of y = alpha * A^T * x + beta * y, where
+       elements in the sparsity pattern of the matrix are treated as "true". */
    void BooleanMultTranspose(int alpha, const int *x, int beta, int *y)
    {
+      HostRead();
       internal::hypre_ParCSRMatrixBooleanMatvecT(A, alpha, const_cast<int*>(x),
                                                  beta, y);
+      HypreRead();
    }
 
    /// Initialize all entries with value.
@@ -592,6 +713,49 @@ public:
    /// Eliminate rows from the diagonal and off-diagonal blocks of the matrix.
    void EliminateRows(const Array<int> &rows);
 
+   /** @brief Eliminate essential BC specified by @a ess_dof_list from the
+       solution @a X to the r.h.s. @a B. */
+   /** This matrix is the matrix with eliminated BC, while @a Ae is such that
+       (A+Ae) is the original (Neumann) matrix before elimination. */
+   void EliminateBC(const HypreParMatrix &Ae, const Array<int> &ess_dof_list,
+                    const Vector &X, Vector &B) const;
+
+   /// Update the internal hypre_ParCSRMatrix object, A, to be on host.
+   /** After this call A's diagonal and off-diagonal should not be modified
+       until after a suitable call to {Host,Hypre}{Write,ReadWrite}. */
+   void HostRead() const { Read(Device::GetHostMemoryClass()); }
+
+   /// Update the internal hypre_ParCSRMatrix object, A, to be on host.
+   /** After this call A's diagonal and off-diagonal can be modified on host
+       and subsequent calls to Hypre{Read,Write,ReadWrite} will require a deep
+       copy of the data if hypre is built with device support. */
+   void HostReadWrite() { ReadWrite(Device::GetHostMemoryClass()); }
+
+   /// Update the internal hypre_ParCSRMatrix object, A, to be on host.
+   /** Similar to HostReadWrite(), except that the data will never be copied
+       from device to host to ensure host contains the correct current data. */
+   void HostWrite() { Write(Device::GetHostMemoryClass()); }
+
+   /** @brief Update the internal hypre_ParCSRMatrix object, A, to be in hypre
+       memory space. */
+   /** After this call A's diagonal and off-diagonal should not be modified
+       until after a suitable call to {Host,Hypre}{Write,ReadWrite}. */
+   void HypreRead() const { Read(GetHypreMemoryClass()); }
+
+   /** @brief Update the internal hypre_ParCSRMatrix object, A, to be in hypre
+       memory space. */
+   /** After this call A's diagonal and off-diagonal can be modified in hypre
+       memory space and subsequent calls to Host{Read,Write,ReadWrite} will
+       require a deep copy of the data if hypre is built with device support. */
+   void HypreReadWrite() { ReadWrite(GetHypreMemoryClass()); }
+
+   /** @brief Update the internal hypre_ParCSRMatrix object, A, to be in hypre
+       memory space. */
+   /** Similar to HostReadWrite(), except that the data will never be copied
+       from host to hypre memory space to ensure the latter contains the correct
+       current data. */
+   void HypreWrite() { Write(GetHypreMemoryClass()); }
+
    /// Prints the locally owned rows in parallel
    void Print(const char *fname, HYPRE_Int offi = 0, HYPRE_Int offj = 0) const;
    /// Reads the matrix from a file
@@ -664,10 +828,11 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
 HypreParMatrix * HypreParMatrixFromBlocks(Array2D<HypreParMatrix*> &blocks,
                                           Array2D<double> *blockCoeff=NULL);
 
-/** Eliminate essential BC specified by 'ess_dof_list' from the solution X to
-    the r.h.s. B. Here A is a matrix with eliminated BC, while Ae is such that
-    (A+Ae) is the original (Neumann) matrix before elimination. */
-void EliminateBC(HypreParMatrix &A, HypreParMatrix &Ae,
+/** @brief Eliminate essential BC specified by @a ess_dof_list from the solution
+    @a X to the r.h.s. @a B. */
+/** Here @a A is a matrix with eliminated BC, while @a Ae is such that (A+Ae) is
+    the original (Neumann) matrix before elimination. */
+void EliminateBC(const HypreParMatrix &A, const HypreParMatrix &Ae,
                  const Array<int> &ess_dof_list, const Vector &X, Vector &B);
 
 
@@ -679,6 +844,10 @@ protected:
    HypreParMatrix *A;
    /// Right-hand side and solution vectors
    mutable HypreParVector *B, *X;
+   /** @brief Auxiliary buffers for the case when the input or output arrays in
+       methods like Mult(const Vector &, Vector &) need to be deep copied in
+       order to be used by hypre. */
+   mutable Memory<double> auxB, auxX;
    /// Temporary vectors
    mutable HypreParVector *V, *Z;
    /// FIR Filter Temporary Vectors
@@ -739,10 +908,15 @@ public:
    enum Type { Jacobi = 0, l1Jacobi = 1, l1GS = 2, l1GStr = 4, lumpedJacobi = 5,
                GS = 6, OPFS = 10, Chebyshev = 16, Taubin = 1001, FIR = 1002
              };
+#ifndef HYPRE_USING_CUDA
+   static constexpr Type default_type = l1GS;
+#else
+   static constexpr Type default_type = l1Jacobi;
+#endif
 
    HypreSmoother();
 
-   HypreSmoother(const HypreParMatrix &A_, int type = l1GS,
+   HypreSmoother(const HypreParMatrix &A_, int type = default_type,
                  int relax_times = 1, double relax_weight = 1.0,
                  double omega = 1.0, int poly_order = 2,
                  double poly_fraction = .3, int eig_est_cg_iter = 10);
@@ -811,6 +985,8 @@ protected:
    /// Right-hand side and solution vector
    mutable HypreParVector *B, *X;
 
+   mutable Memory<double> auxB, auxX;
+
    /// Was hypre's Setup function called already?
    mutable int setup_called;
 
@@ -833,6 +1009,8 @@ public:
    virtual void SetOperator(const Operator &op)
    { mfem_error("HypreSolvers do not support SetOperator!"); }
 
+   virtual MemoryClass GetMemoryClass() const { return GetHypreMemoryClass(); }
+
    /// Solve the linear system Ax=b
    virtual void Mult(const HypreParVector &b, HypreParVector &x) const;
    virtual void Mult(const Vector &b, Vector &x) const;
@@ -1251,7 +1429,7 @@ public:
        See "Nonsymmetric Algebraic Multigrid Based on Local Approximate Ideal
        Restriction (AIR)," Manteuffel, Ruge, Southworth, SISC (2018),
        DOI:/10.1137/17M1144350. Options: "distanceR" -> distance of neighbor
-       DOFs to buld restriction operator; options include 1, 2, and 15 (1.5).
+       DOFs for the restriction operator; options include 1, 2, and 15 (1.5).
        Strings "prerelax" and "postrelax" indicate points to relax on:
        F = F-points, C = C-points, A = all points. E.g., FFC -> relax on
        F-points, relax again on F-points, then relax on C-points. */

linalg/hypre_parcsr.cpp

@@ -11,10 +11,12 @@
 
 #include "../config/config.hpp"
 #include "../general/error.hpp"
+#include "../general/forall.hpp"
 
 #ifdef MFEM_USE_MPI
 
 #include "hypre_parcsr.hpp"
+#include "hypre.hpp"
 #include <limits>
 #include <cmath>
 
@@ -208,8 +210,8 @@ void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A,
    HYPRE_Int index, start;
    HYPRE_Int i, j, k, irow;
 
-   HYPRE_Real *eliminate_row = mfem_hypre_CTAlloc(HYPRE_Real, diag_nrows);
-   HYPRE_Real *eliminate_col = mfem_hypre_CTAlloc(HYPRE_Real, offd_ncols);
+   HYPRE_Real *eliminate_row = mfem_hypre_CTAlloc_host(HYPRE_Real, diag_nrows);
+   HYPRE_Real *eliminate_col = mfem_hypre_CTAlloc_host(HYPRE_Real, offd_ncols);
    HYPRE_Real *buf_data, coef;
 
    /* make sure A has a communication package */
@@ -236,9 +238,9 @@ void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A,
    /* use a Matvec communication pattern to find (in eliminate_col)
       which of the local offd columns are to be eliminated */
    num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-   buf_data = mfem_hypre_CTAlloc(HYPRE_Real,
-                                 hypre_ParCSRCommPkgSendMapStart(comm_pkg,
-                                                                 num_sends));
+   buf_data = mfem_hypre_CTAlloc_host(
+                 HYPRE_Real,
+                 hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
    index = 0;
    for (i = 0; i < num_sends; i++)
    {
@@ -270,8 +272,9 @@ void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A,
       }
    }
 
-   offd_cols_to_elim = mfem_hypre_CTAlloc(HYPRE_Int, num_offd_cols_to_elim);
-   eliminate_coefs = mfem_hypre_CTAlloc(HYPRE_Real, num_offd_cols_to_elim);
+   offd_cols_to_elim = mfem_hypre_CTAlloc_host(HYPRE_Int,
+                                               num_offd_cols_to_elim);
+   eliminate_coefs = mfem_hypre_CTAlloc_host(HYPRE_Real, num_offd_cols_to_elim);
 
    /* get a list of offd column indices and coefs */
    num_offd_cols_to_elim = 0;
@@ -286,9 +289,9 @@ void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A,
       }
    }
 
-   mfem_hypre_TFree(buf_data);
-   mfem_hypre_TFree(eliminate_col);
-   mfem_hypre_TFree(eliminate_row);
+   mfem_hypre_TFree_host(buf_data);
+   mfem_hypre_TFree_host(eliminate_col);
+   mfem_hypre_TFree_host(eliminate_row);
 
    /* eliminate the off-diagonal part */
    hypre_CSRMatrixEliminateOffdColsAXB(offd, num_offd_cols_to_elim,
@@ -305,8 +308,8 @@ void hypre_ParCSRMatrixEliminateAXB(hypre_ParCSRMatrix *A,
       Bdata[irow] = Xdata[irow];
    }
 
-   mfem_hypre_TFree(offd_cols_to_elim);
-   mfem_hypre_TFree(eliminate_coefs);
+   mfem_hypre_TFree_host(offd_cols_to_elim);
+   mfem_hypre_TFree_host(eliminate_coefs);
 }
 
 
@@ -332,7 +335,7 @@ void hypre_CSRMatrixElimCreate(hypre_CSRMatrix *A,
    HYPRE_Int  *A_j     = hypre_CSRMatrixJ(A);
    HYPRE_Int   A_rows  = hypre_CSRMatrixNumRows(A);
 
-   hypre_CSRMatrixI(Ae) = mfem_hypre_TAlloc(HYPRE_Int, A_rows+1);
+   hypre_CSRMatrixI(Ae) = mfem_hypre_TAlloc_host(HYPRE_Int, A_rows+1);
 
    HYPRE_Int  *Ae_i    = hypre_CSRMatrixI(Ae);
    HYPRE_Int   nnz     = 0;
@@ -373,9 +376,12 @@ void hypre_CSRMatrixElimCreate(hypre_CSRMatrix *A,
    }
    Ae_i[A_rows] = nnz;
 
-   hypre_CSRMatrixJ(Ae) = mfem_hypre_TAlloc(HYPRE_Int, nnz);
-   hypre_CSRMatrixData(Ae) = mfem_hypre_TAlloc(HYPRE_Real, nnz);
+   hypre_CSRMatrixJ(Ae) = mfem_hypre_TAlloc_host(HYPRE_Int, nnz);
+   hypre_CSRMatrixData(Ae) = mfem_hypre_TAlloc_host(HYPRE_Real, nnz);
    hypre_CSRMatrixNumNonzeros(Ae) = nnz;
+#if MFEM_HYPRE_VERSION >= 21800
+   hypre_CSRMatrixMemoryLocation(Ae) = HYPRE_MEMORY_HOST;
+#endif
 }
 
 /*
@@ -481,6 +487,7 @@ void hypre_CSRMatrixEliminateRows(hypre_CSRMatrix *A,
                                          \ A_bi | A_bb - I /
 
 */
+
 void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
                                     hypre_ParCSRMatrix **Ae,
                                     HYPRE_Int num_rowscols_to_elim,
@@ -524,8 +531,10 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       HYPRE_Int num_sends, *int_buf_data;
       HYPRE_Int index, start;
 
-      HYPRE_Int *eliminate_diag_col = mfem_hypre_CTAlloc(HYPRE_Int, A_diag_ncols);
-      HYPRE_Int *eliminate_offd_col = mfem_hypre_CTAlloc(HYPRE_Int, A_offd_ncols);
+      HYPRE_Int *eliminate_diag_col = mfem_hypre_CTAlloc_host(HYPRE_Int,
+                                                              A_diag_ncols);
+      HYPRE_Int *eliminate_offd_col = mfem_hypre_CTAlloc_host(HYPRE_Int,
+                                                              A_offd_ncols);
 
       /* make sure A has a communication package */
       comm_pkg = hypre_ParCSRMatrixCommPkg(A);
@@ -548,7 +557,7 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       /* use a Matvec communication pattern to find (in eliminate_col)
          which of the local offd columns are to be eliminated */
       num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-      int_buf_data = mfem_hypre_CTAlloc(
+      int_buf_data = mfem_hypre_CTAlloc_host(
                         HYPRE_Int,
                         hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
       index = 0;
@@ -562,7 +571,8 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
          }
       }
       comm_handle = hypre_ParCSRCommHandleCreate(11, comm_pkg,
-                                                 int_buf_data, eliminate_offd_col);
+                                                 int_buf_data,
+                                                 eliminate_offd_col);
 
       /* eliminate diagonal part, overlapping it with communication */
       if (ignore_rows)
@@ -602,7 +612,8 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
          if (eliminate_offd_col[i]) { num_offd_cols_to_elim++; }
       }
 
-      offd_cols_to_elim = mfem_hypre_CTAlloc(HYPRE_Int, num_offd_cols_to_elim);
+      offd_cols_to_elim = mfem_hypre_CTAlloc_host(HYPRE_Int,
+                                                  num_offd_cols_to_elim);
 
       /* get a list of offd column indices and coefs */
       num_offd_cols_to_elim = 0;
@@ -614,14 +625,14 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
          }
       }
 
-      mfem_hypre_TFree(int_buf_data);
-      mfem_hypre_TFree(eliminate_offd_col);
-      mfem_hypre_TFree(eliminate_diag_col);
+      mfem_hypre_TFree_host(int_buf_data);
+      mfem_hypre_TFree_host(eliminate_offd_col);
+      mfem_hypre_TFree_host(eliminate_diag_col);
    }
 
    /* eliminate the off-diagonal part */
-   col_mark = mfem_hypre_CTAlloc(HYPRE_Int, A_offd_ncols);
-   col_remap = mfem_hypre_CTAlloc(HYPRE_Int, A_offd_ncols);
+   col_mark = mfem_hypre_CTAlloc_host(HYPRE_Int, A_offd_ncols);
+   col_remap = mfem_hypre_CTAlloc_host(HYPRE_Int, A_offd_ncols);
 
    if (ignore_rows)
    {
@@ -665,7 +676,7 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
       if (col_mark[i]) { Ae_offd_ncols++; }
    }
 
-   Ae_col_map_offd = mfem_hypre_CTAlloc(HYPRE_BigInt, Ae_offd_ncols);
+   Ae_col_map_offd = mfem_hypre_CTAlloc_host(HYPRE_BigInt, Ae_offd_ncols);
 
    Ae_offd_ncols = 0;
    for (i = 0; i < A_offd_ncols; i++)
@@ -679,9 +690,9 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
    hypre_ParCSRMatrixColMapOffd(*Ae) = Ae_col_map_offd;
    hypre_CSRMatrixNumCols(Ae_offd) = Ae_offd_ncols;
 
-   mfem_hypre_TFree(col_remap);
-   mfem_hypre_TFree(col_mark);
-   mfem_hypre_TFree(offd_cols_to_elim);
+   mfem_hypre_TFree_host(col_remap);
+   mfem_hypre_TFree_host(col_mark);
+   mfem_hypre_TFree_host(offd_cols_to_elim);
 
    hypre_ParCSRMatrixSetNumNonzeros(*Ae);
    hypre_MatvecCommPkgCreate(*Ae);
@@ -718,11 +729,11 @@ void hypre_CSRMatrixSplit(hypre_CSRMatrix *A,
    HYPRE_Int A_rows = hypre_CSRMatrixNumRows(A);
    HYPRE_Int A_cols = hypre_CSRMatrixNumCols(A);
 
-   HYPRE_Int *num_rows = mfem_hypre_CTAlloc(HYPRE_Int, nr);
-   HYPRE_Int *num_cols = mfem_hypre_CTAlloc(HYPRE_Int, nc);
+   HYPRE_Int *num_rows = mfem_hypre_CTAlloc_host(HYPRE_Int, nr);
+   HYPRE_Int *num_cols = mfem_hypre_CTAlloc_host(HYPRE_Int, nc);
 
-   HYPRE_Int *block_row = mfem_hypre_TAlloc(HYPRE_Int, A_rows);
-   HYPRE_Int *block_col = mfem_hypre_TAlloc(HYPRE_Int, A_cols);
+   HYPRE_Int *block_row = mfem_hypre_TAlloc_host(HYPRE_Int, A_rows);
+   HYPRE_Int *block_col = mfem_hypre_TAlloc_host(HYPRE_Int, A_cols);
 
    for (i = 0; i < A_rows; i++)
    {
@@ -738,8 +749,13 @@ void hypre_CSRMatrixSplit(hypre_CSRMatrix *A,
    {
       for (j = 0; j < nc; j++)
       {
-         hypre_CSRMatrix *B = hypre_CSRMatrixCreate(num_rows[i], num_cols[j], 0);
-         hypre_CSRMatrixI(B) = mfem_hypre_CTAlloc(HYPRE_Int, num_rows[i] + 1);
+         hypre_CSRMatrix *B = hypre_CSRMatrixCreate(num_rows[i],
+                                                    num_cols[j], 0);
+         hypre_CSRMatrixI(B) = mfem_hypre_CTAlloc_host(HYPRE_Int,
+                                                       num_rows[i] + 1);
+#if MFEM_HYPRE_VERSION >= 21800
+         hypre_CSRMatrixMemoryLocation(B) = HYPRE_MEMORY_HOST;
+#endif
          blocks[i*nc + j] = B;
       }
    }
@@ -768,8 +784,8 @@ void hypre_CSRMatrixSplit(hypre_CSRMatrix *A,
          rs = B_i[k], B_i[k] = nnz, nnz += rs;
       }
 
-      hypre_CSRMatrixJ(B) = mfem_hypre_TAlloc(HYPRE_Int, nnz);
-      hypre_CSRMatrixData(B) = mfem_hypre_TAlloc(HYPRE_Complex, nnz);
+      hypre_CSRMatrixJ(B) = mfem_hypre_TAlloc_host(HYPRE_Int, nnz);
+      hypre_CSRMatrixData(B) = mfem_hypre_TAlloc_host(HYPRE_Complex, nnz);
       hypre_CSRMatrixNumNonzeros(B) = nnz;
    }
 
@@ -789,11 +805,11 @@ void hypre_CSRMatrixSplit(hypre_CSRMatrix *A,
       }
    }
 
-   mfem_hypre_TFree(block_col);
-   mfem_hypre_TFree(block_row);
+   mfem_hypre_TFree_host(block_col);
+   mfem_hypre_TFree_host(block_row);
 
-   mfem_hypre_TFree(num_cols);
-   mfem_hypre_TFree(num_rows);
+   mfem_hypre_TFree_host(num_cols);
+   mfem_hypre_TFree_host(num_rows);
 }
 
 
@@ -824,8 +840,8 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
    HYPRE_Int num_blocks = nr * nc;
 
    /* mark local rows and columns with block number */
-   HYPRE_Int *row_block_num = mfem_hypre_TAlloc(HYPRE_Int, local_rows);
-   HYPRE_Int *col_block_num = mfem_hypre_TAlloc(HYPRE_Int, local_cols);
+   HYPRE_Int *row_block_num = mfem_hypre_TAlloc_host(HYPRE_Int, local_rows);
+   HYPRE_Int *col_block_num = mfem_hypre_TAlloc_host(HYPRE_Int, local_cols);
 
    for (i = 0; i < local_rows; i++)
    {
@@ -837,7 +853,8 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
    }
 
    /* determine the block numbers for offd columns */
-   HYPRE_BigInt* offd_col_block_num = mfem_hypre_TAlloc(HYPRE_BigInt, offd_cols);
+   HYPRE_BigInt *offd_col_block_num = mfem_hypre_TAlloc_host(HYPRE_BigInt,
+                                                             offd_cols);
    hypre_ParCSRCommHandle *comm_handle;
    HYPRE_BigInt *int_buf_data;
    {
@@ -850,18 +867,19 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
       }
 
       /* calculate the final global column numbers for each block */
-      HYPRE_Int *count = mfem_hypre_CTAlloc(HYPRE_Int, nc);
-      HYPRE_BigInt *block_global_col = mfem_hypre_TAlloc(HYPRE_BigInt, local_cols);
+      HYPRE_Int *count = mfem_hypre_CTAlloc_host(HYPRE_Int, nc);
+      HYPRE_BigInt *block_global_col = mfem_hypre_TAlloc_host(HYPRE_BigInt,
+                                                              local_cols);
       HYPRE_BigInt first_col = hypre_ParCSRMatrixFirstColDiag(A) / nc;
       for (i = 0; i < local_cols; i++)
       {
          block_global_col[i] = first_col + count[col_block_num[i]]++;
       }
-      mfem_hypre_TFree(count);
+      mfem_hypre_TFree_host(count);
 
       /* use a Matvec communication pattern to determine offd_col_block_num */
       HYPRE_Int num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-      int_buf_data = mfem_hypre_CTAlloc(
+      int_buf_data = mfem_hypre_CTAlloc_host(
                         HYPRE_BigInt,
                         hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
       HYPRE_Int start, index = 0;
@@ -874,7 +892,8 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
             int_buf_data[index++] = col_block_num[k] + nc*block_global_col[k];
          }
       }
-      mfem_hypre_TFree(block_global_col);
+
+      mfem_hypre_TFree_host(block_global_col);
 
 #if MFEM_HYPRE_VERSION < 21600
       const int job = 11;
@@ -892,8 +911,8 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
       hypre_MPI_Comm_size(comm, &num_procs);
    }
 
-   HYPRE_BigInt *row_starts = mfem_hypre_TAlloc(HYPRE_BigInt, num_procs+1);
-   HYPRE_BigInt *col_starts = mfem_hypre_TAlloc(HYPRE_BigInt, num_procs+1);
+   HYPRE_BigInt *row_starts = mfem_hypre_TAlloc_host(HYPRE_BigInt, num_procs+1);
+   HYPRE_BigInt *col_starts = mfem_hypre_TAlloc_host(HYPRE_BigInt, num_procs+1);
    for (i = 0; i <= num_procs; i++)
    {
       row_starts[i] = hypre_ParCSRMatrixRowStarts(A)[i] / nr;
@@ -908,30 +927,33 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
    }
 
    /* split diag part */
-   hypre_CSRMatrix **csr_blocks = mfem_hypre_TAlloc(hypre_CSRMatrix*, nr*nc);
+   hypre_CSRMatrix **csr_blocks = mfem_hypre_TAlloc_host(hypre_CSRMatrix*,
+                                                         nr*nc);
    hypre_CSRMatrixSplit(Adiag, nr, nc, row_block_num, col_block_num,
                         csr_blocks);
 
    for (i = 0; i < num_blocks; i++)
    {
-      mfem_hypre_TFree(hypre_ParCSRMatrixDiag(blocks[i]));
+      mfem_hypre_TFree_host(hypre_ParCSRMatrixDiag(blocks[i]));
       hypre_ParCSRMatrixDiag(blocks[i]) = csr_blocks[i];
    }
 
    /* finish communication, receive offd_col_block_num */
    hypre_ParCSRCommHandleDestroy(comm_handle);
-   mfem_hypre_TFree(int_buf_data);
+   mfem_hypre_TFree_host(int_buf_data);
 
    /* decode global offd column numbers */
-   HYPRE_Int *offd_col_block_num_nc = mfem_hypre_TAlloc(HYPRE_Int, offd_cols);
-   HYPRE_BigInt* offd_global_col = mfem_hypre_TAlloc(HYPRE_BigInt, offd_cols);
+   HYPRE_Int *offd_col_block_num_nc = mfem_hypre_TAlloc_host(HYPRE_Int,
+                                                             offd_cols);
+   HYPRE_BigInt* offd_global_col = mfem_hypre_TAlloc_host(HYPRE_BigInt,
+                                                          offd_cols);
    for (i = 0; i < offd_cols; i++)
    {
       offd_global_col[i] = offd_col_block_num[i] / nc;
       offd_col_block_num_nc[i] = offd_col_block_num[i] % nc;
    }
 
-   mfem_hypre_TFree(offd_col_block_num);
+   mfem_hypre_TFree_host(offd_col_block_num);
 
    /* split offd part */
    hypre_CSRMatrixSplit(Aoffd, nr, nc, row_block_num, offd_col_block_num_nc,
@@ -939,13 +961,13 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
 
    for (i = 0; i < num_blocks; i++)
    {
-      mfem_hypre_TFree(hypre_ParCSRMatrixOffd(blocks[i]));
+      mfem_hypre_TFree_host(hypre_ParCSRMatrixOffd(blocks[i]));
       hypre_ParCSRMatrixOffd(blocks[i]) = csr_blocks[i];
    }
 
-   mfem_hypre_TFree(csr_blocks);
-   mfem_hypre_TFree(col_block_num);
-   mfem_hypre_TFree(row_block_num);
+   mfem_hypre_TFree_host(csr_blocks);
+   mfem_hypre_TFree_host(col_block_num);
+   mfem_hypre_TFree_host(row_block_num);
 
    /* update block col-maps */
    for (int bi = 0; bi < nr; bi++)
@@ -956,8 +978,8 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
          hypre_CSRMatrix *block_offd = hypre_ParCSRMatrixOffd(block);
          HYPRE_Int block_offd_cols = hypre_CSRMatrixNumCols(block_offd);
 
-         HYPRE_BigInt *block_col_map = mfem_hypre_TAlloc(HYPRE_BigInt,
-                                                         block_offd_cols);
+         HYPRE_BigInt *block_col_map = mfem_hypre_TAlloc_host(HYPRE_BigInt,
+                                                              block_offd_cols);
          for (i = j = 0; i < offd_cols; i++)
          {
             HYPRE_Int bn = offd_col_block_num_nc[i];
@@ -969,8 +991,8 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
       }
    }
 
-   mfem_hypre_TFree(offd_global_col);
-   mfem_hypre_TFree(offd_col_block_num_nc);
+   mfem_hypre_TFree_host(offd_global_col);
+   mfem_hypre_TFree_host(offd_col_block_num_nc);
 
    /* finish the new matrices, make them own all the stuff */
    for (i = 0; i < num_blocks; i++)
@@ -1367,7 +1389,7 @@ hypre_ParCSRCommHandleCreate_bool(HYPRE_Int            job,
    HYPRE_Int                  ip, vec_start, vec_len;
 
    num_requests = num_sends + num_recvs;
-   requests = mfem_hypre_CTAlloc(hypre_MPI_Request, num_requests);
+   requests = mfem_hypre_CTAlloc_host(hypre_MPI_Request, num_requests);
 
    hypre_MPI_Comm_size(comm, &num_procs);
    hypre_MPI_Comm_rank(comm, &my_id);
@@ -1424,7 +1446,7 @@ hypre_ParCSRCommHandleCreate_bool(HYPRE_Int            job,
     * set up comm_handle and return
     *--------------------------------------------------------------------*/
 
-   comm_handle = mfem_hypre_CTAlloc(hypre_ParCSRCommHandle, 1);
+   comm_handle = mfem_hypre_CTAlloc_host(hypre_ParCSRCommHandle, 1);
 
    hypre_ParCSRCommHandleCommPkg(comm_handle)     = comm_pkg;
    hypre_ParCSRCommHandleSendData(comm_handle)    = send_data;
@@ -1452,7 +1474,7 @@ void hypre_ParCSRMatrixAbsMatvec(hypre_ParCSRMatrix *A,
 
    HYPRE_Real        *x_tmp, *x_buf;
 
-   x_tmp = mfem_hypre_CTAlloc(HYPRE_Real, num_cols_offd);
+   x_tmp = mfem_hypre_CTAlloc_host(HYPRE_Real, num_cols_offd);
 
    /*---------------------------------------------------------------------
     * If there exists no CommPkg for A, a CommPkg is generated using
@@ -1465,7 +1487,7 @@ void hypre_ParCSRMatrixAbsMatvec(hypre_ParCSRMatrix *A,
    }
 
    num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-   x_buf = mfem_hypre_CTAlloc(
+   x_buf = mfem_hypre_CTAlloc_host(
               HYPRE_Real, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
 
    index = 0;
@@ -1489,8 +1511,8 @@ void hypre_ParCSRMatrixAbsMatvec(hypre_ParCSRMatrix *A,
       hypre_CSRMatrixAbsMatvec(offd, alpha, x_tmp, 1.0, y);
    }
 
-   mfem_hypre_TFree(x_buf);
-   mfem_hypre_TFree(x_tmp);
+   mfem_hypre_TFree_host(x_buf);
+   mfem_hypre_TFree_host(x_tmp);
 }
 
 /* Based on hypre_ParCSRMatrixMatvecT in par_csr_matvec.c */
@@ -1511,7 +1533,7 @@ void hypre_ParCSRMatrixAbsMatvecT(hypre_ParCSRMatrix *A,
 
    HYPRE_Int               i, j, jj, end, num_sends;
 
-   y_tmp = mfem_hypre_TAlloc(HYPRE_Real, num_cols_offd);
+   y_tmp = mfem_hypre_TAlloc_host(HYPRE_Real, num_cols_offd);
 
    /*---------------------------------------------------------------------
     * If there exists no CommPkg for A, a CommPkg is generated using
@@ -1524,7 +1546,7 @@ void hypre_ParCSRMatrixAbsMatvecT(hypre_ParCSRMatrix *A,
    }
 
    num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-   y_buf = mfem_hypre_CTAlloc(
+   y_buf = mfem_hypre_CTAlloc_host(
               HYPRE_Real, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
 
    if (num_cols_offd)
@@ -1568,8 +1590,8 @@ void hypre_ParCSRMatrixAbsMatvecT(hypre_ParCSRMatrix *A,
       }
    }
 
-   mfem_hypre_TFree(y_buf);
-   mfem_hypre_TFree(y_tmp);
+   mfem_hypre_TFree_host(y_buf);
+   mfem_hypre_TFree_host(y_tmp);
 }
 
 /* Based on hypre_ParCSRMatrixMatvec in par_csr_matvec.c */
@@ -1589,7 +1611,7 @@ void hypre_ParCSRMatrixBooleanMatvec(hypre_ParCSRMatrix *A,
 
    HYPRE_Bool        *x_tmp, *x_buf;
 
-   x_tmp = mfem_hypre_CTAlloc(HYPRE_Bool, num_cols_offd);
+   x_tmp = mfem_hypre_CTAlloc_host(HYPRE_Bool, num_cols_offd);
 
    /*---------------------------------------------------------------------
     * If there exists no CommPkg for A, a CommPkg is generated using
@@ -1602,7 +1624,7 @@ void hypre_ParCSRMatrixBooleanMatvec(hypre_ParCSRMatrix *A,
    }
 
    num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-   x_buf = mfem_hypre_CTAlloc(
+   x_buf = mfem_hypre_CTAlloc_host(
               HYPRE_Bool, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
 
    index = 0;
@@ -1626,8 +1648,8 @@ void hypre_ParCSRMatrixBooleanMatvec(hypre_ParCSRMatrix *A,
       hypre_CSRMatrixBooleanMatvec(offd, alpha, x_tmp, 1, y);
    }
 
-   mfem_hypre_TFree(x_buf);
-   mfem_hypre_TFree(x_tmp);
+   mfem_hypre_TFree_host(x_buf);
+   mfem_hypre_TFree_host(x_tmp);
 }
 
 /* Based on hypre_ParCSRMatrixMatvecT in par_csr_matvec.c */
@@ -1648,7 +1670,7 @@ void hypre_ParCSRMatrixBooleanMatvecT(hypre_ParCSRMatrix *A,
 
    HYPRE_Int               i, j, jj, end, num_sends;
 
-   y_tmp = mfem_hypre_TAlloc(HYPRE_Bool, num_cols_offd);
+   y_tmp = mfem_hypre_TAlloc_host(HYPRE_Bool, num_cols_offd);
 
    /*---------------------------------------------------------------------
     * If there exists no CommPkg for A, a CommPkg is generated using
@@ -1661,7 +1683,7 @@ void hypre_ParCSRMatrixBooleanMatvecT(hypre_ParCSRMatrix *A,
    }
 
    num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-   y_buf = mfem_hypre_CTAlloc(
+   y_buf = mfem_hypre_CTAlloc_host(
               HYPRE_Bool, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
 
    if (num_cols_offd)
@@ -1705,8 +1727,8 @@ void hypre_ParCSRMatrixBooleanMatvecT(hypre_ParCSRMatrix *A,
       }
    }
 
-   mfem_hypre_TFree(y_buf);
-   mfem_hypre_TFree(y_tmp);
+   mfem_hypre_TFree_host(y_buf);
+   mfem_hypre_TFree_host(y_tmp);
 }
 
 HYPRE_Int
@@ -1733,7 +1755,7 @@ hypre_CSRMatrixSum(hypre_CSRMatrix *A,
       return -1; /* error: incompatible matrix dimensions */
    }
 
-   marker = mfem_hypre_CTAlloc(HYPRE_Int, ncols_A);
+   marker = mfem_hypre_CTAlloc_host(HYPRE_Int, ncols_A);
    for (ia = 0; ia < ncols_A; ia++)
    {
       marker[ia] = -1;
@@ -1757,7 +1779,7 @@ hypre_CSRMatrixSum(hypre_CSRMatrix *A,
       }
    }
 
-   mfem_hypre_TFree(marker);
+   mfem_hypre_TFree_host(marker);
    return 0;
 }
 
@@ -1818,7 +1840,7 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,
          return NULL; /* error: A_offd and B_offd have different dimensions */
       }
       /* copy A_cmap -> C_cmap */
-      C_cmap = mfem_hypre_TAlloc(HYPRE_BigInt, A_cmap_size);
+      C_cmap = mfem_hypre_TAlloc_host(HYPRE_BigInt, A_cmap_size);
       for (im = 0; im < A_cmap_size; im++)
       {
          C_cmap[im] = A_cmap[im];
@@ -1859,7 +1881,7 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,
       csr_B = hypre_MergeDiagAndOffd(B);
 
       /* add A and B */
-      csr_C_temp = hypre_CSRMatrixAdd(csr_A,csr_B);
+      csr_C_temp = hypre_CSRMatrixAdd(csr_A, csr_B);
 
       /* delete CSR versions of A and B */
       ierr += hypre_CSRMatrixDestroy(csr_A);

linalg/hypre_parcsr.hpp

@@ -19,6 +19,7 @@
 // Enable internal hypre timing routines
 #define HYPRE_TIMING
 
+#include "../general/mem_manager.hpp"
 #include "_hypre_parcsr_mv.h"
 
 // Older hypre versions do not define HYPRE_BigInt and HYPRE_MPI_BIG_INT, so we
@@ -38,13 +39,23 @@ typedef HYPRE_Int HYPRE_BigInt;
 #define mfem_hypre_CTAlloc(type, size) hypre_CTAlloc(type, size)
 #define mfem_hypre_TFree(ptr) hypre_TFree(ptr)
 
+#define mfem_hypre_TAlloc_host(type, size) hypre_TAlloc(type, size)
+#define mfem_hypre_CTAlloc_host(type, size) hypre_CTAlloc(type, size)
+#define mfem_hypre_TFree_host(ptr) hypre_TFree(ptr)
+
 #else // MFEM_HYPRE_VERSION >= 21400
 
 #define mfem_hypre_TAlloc(type, size) \
-   hypre_TAlloc(type, size, HYPRE_MEMORY_HOST)
+   hypre_TAlloc(type, size, HYPRE_MEMORY_DEVICE)
 #define mfem_hypre_CTAlloc(type, size) \
+   hypre_CTAlloc(type, size, HYPRE_MEMORY_DEVICE)
+#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, HYPRE_MEMORY_DEVICE)
+
+#define mfem_hypre_TAlloc_host(type, size) \
+   hypre_TAlloc(type, size, HYPRE_MEMORY_HOST)
+#define mfem_hypre_CTAlloc_host(type, size) \
    hypre_CTAlloc(type, size, HYPRE_MEMORY_HOST)
-#define mfem_hypre_TFree(ptr) hypre_TFree(ptr, HYPRE_MEMORY_HOST)
+#define mfem_hypre_TFree_host(ptr) hypre_TFree(ptr, HYPRE_MEMORY_HOST)
 
 // Notes regarding allocation and deallocation of hypre objects in 2.14.0
 //-----------------------------------------------------------------------
@@ -85,6 +96,12 @@ namespace mfem
 // become part of HYPRE at some point. In the meantime the module can be
 // thought of as an extension of HYPRE.
 
+struct MemoryIJData
+{
+   Memory<HYPRE_Int> I, J;
+   Memory<double> data;
+};
+
 namespace internal
 {
 

linalg/linalg.hpp

@@ -40,6 +40,10 @@
 #include "sundials.hpp"
 #endif
 
+#ifdef MFEM_USE_EPIC
+#include "epic.hpp"
+#endif
+
 #ifdef MFEM_USE_HIOP
 #include "hiop.hpp"
 #endif

linalg/operator.cpp

@@ -33,7 +33,7 @@ void Operator::InitTVectors(const Operator *Po, const Operator *Ri,
    else
    {
       // B points to same data as b
-      B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+      B.MakeRef(b, 0, b.Size());
    }
    if (!IsIdentityProlongation(Pi))
    {
@@ -44,7 +44,7 @@ void Operator::InitTVectors(const Operator *Po, const Operator *Ri,
    else
    {
       // X points to same data as x
-      X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
+      X.MakeRef(x, 0, x.Size());
    }
 }
 

linalg/solvers.cpp

@@ -606,9 +606,11 @@ void SLI(const Operator &A, Solver &B, const Vector &b, Vector &x,
 
 void CGSolver::UpdateVectors()
 {
-   r.SetSize(width);
-   d.SetSize(width);
-   z.SetSize(width);
+   MemoryType mt = GetMemoryType(oper->GetMemoryClass());
+
+   r.SetSize(width, mt); r.UseDevice(true);
+   d.SetSize(width, mt); d.UseDevice(true);
+   z.SetSize(width, mt); z.UseDevice(true);
 }
 
 void CGSolver::Mult(const Vector &b, Vector &x) const
@@ -616,6 +618,7 @@ void CGSolver::Mult(const Vector &b, Vector &x) const
    int i;
    double r0, den, nom, nom0, betanom, alpha, beta;
 
+   x.UseDevice(true);
    if (iterative_mode)
    {
       oper->Mult(x, r);

linalg/sundials.cpp

@@ -425,6 +425,26 @@ static SUNMatrix_ID MatGetID(SUNMatrix)
    return (SUNMATRIX_CUSTOM);
 }
 
+//BEGIN WORKAROUND CODE
+static SUNMatrix MatClone(SUNMatrix A)
+{
+   SUNMatrix B = SUNMatNewEmpty();
+   MFEM_VERIFY(B, "error in MatClone()");
+
+   B->content = A->content;
+
+   int flag = SUNMatCopyOps(A, B);
+   MFEM_VERIFY(flag == SUNMAT_SUCCESS, "error in MatClone()");
+
+   return B;
+}
+
+static int MatCopy(SUNMatrix, SUNMatrix)
+{
+   return 0;
+}
+// END WORKAROUN CODE
+
 static void MatDestroy(SUNMatrix A)
 {
    if (A->content) { A->content = NULL; }
@@ -1493,7 +1513,12 @@ void ARKStepSolver::UseMFEMMassLinearSolver(int tdep)
    MFEM_VERIFY(M, "error in SUNMatNewEmpty()");
 
    M->content      = this;
-   M->ops->getid   = SUNMatGetID;
+// BEGIN WORKAROUND CODE
+   // M->ops->getid   = SUNMatGetID;
+   M->ops->getid = MatGetID;
+   M->ops->clone = MatClone;
+   M->ops->copy = MatCopy;
+// END WORKAROUND CODE
    M->ops->matvec  = ARKStepSolver::MassMult1;
    M->ops->destroy = MatDestroy;
 
@@ -1537,6 +1562,12 @@ void ARKStepSolver::SetSStolerances(double reltol, double abstol)
    MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSStolerances()");
 }
 
+void ARKStepSolver::SetMaxIter(int iterations)
+{
+  flag = ARKStepSetMaxNonlinIters(sundials_mem, iterations);
+  MFEM_VERIFY(flag == ARK_SUCCESS, "error in ARKStepSetMaxNonlinIters()");
+}
+
 void ARKStepSolver::SetMaxStep(double dt_max)
 {
    flag = ARKStepSetMaxStep(sundials_mem, dt_max);

linalg/sundials.hpp

@@ -666,6 +666,9 @@ public:
    /// Set the scalar relative and scalar absolute tolerances.
    void SetSStolerances(double reltol, double abstol);
 
+   /// Set the maximum number of nonlinear iterations per RK stage
+   void SetMaxIter(int iterations);
+
    /// Set the maximum time step.
    void SetMaxStep(double dt_max);
 

linalg/superlu.cpp

@@ -189,6 +189,9 @@ SuperLURowLocMatrix::SuperLURowLocMatrix( const HypreParMatrix & hypParMat )
    dCreate_CompRowLoc_Matrix_dist(A, m, n, nnz_loc, m_loc, fst_row,
                                   nzval, colind, rowptr,
                                   SLU_NR_loc, SLU_D, SLU_GE);
+
+   // Save global number of columns (width) of the matrix
+   num_global_cols = n;
 }
 
 SuperLURowLocMatrix::~SuperLURowLocMatrix()
@@ -262,7 +265,7 @@ SuperLUSolver::~SuperLUSolver()
    if ( LUStructInitialized_ )
    {
       ScalePermstructFree(SPstruct);
-      Destroy_LU(width, grid, LUstruct);
+      Destroy_LU(APtr_->GetGlobalNumColumns(), grid, LUstruct);
       LUstructFree(LUstruct);
    }
 

linalg/superlu.hpp

@@ -72,9 +72,12 @@ public:
 
    void * InternalData() const { return rowLocPtr_; }
 
+   HYPRE_BigInt GetGlobalNumColumns() const { return num_global_cols; }
+
 private:
    MPI_Comm   comm_;
    void     * rowLocPtr_;
+   HYPRE_BigInt num_global_cols;
 
 }; // mfem::SuperLURowLocMatrix
 

linalg/vector.cpp

@@ -773,6 +773,7 @@ void Vector::Randomize(int seed)
    // srand(seed++);
    srand((unsigned)seed);
 
+   HostWrite();
    for (int i = 0; i < size; i++)
    {
       data[i] = std::abs(rand()/max);

linalg/vector.hpp

@@ -159,6 +159,11 @@ public:
    /// Reset the Vector to use the given external Memory @a mem and size @a s.
    /** If @a own_mem is false, the Vector will not own any of the pointers of
        @a mem.
+
+       Note that when @a own_mem is true, the @a mem object can be destroyed
+       immediately by the caller but `mem.Delete()` should NOT be called since
+       the Vector object takes ownership of all pointers owned by @a mem.
+
        @sa NewDataAndSize(). */
    inline void NewMemoryAndSize(const Memory<double> &mem, int s, bool own_mem);
 
@@ -224,10 +229,10 @@ public:
    const Memory<double> &GetMemory() const { return data; }
 
    /// Update the memory location of the vector to match @a v.
-   void SyncMemory(const Vector &v) { GetMemory().Sync(v.GetMemory()); }
+   void SyncMemory(const Vector &v) const { GetMemory().Sync(v.GetMemory()); }
 
    /// Update the alias memory location of the vector to match @a v.
-   void SyncAliasMemory(const Vector &v)
+   void SyncAliasMemory(const Vector &v) const
    { GetMemory().SyncAlias(v.GetMemory(),Size()); }
 
    /// Read the Vector data (host pointer) ownership flag.
@@ -559,8 +564,14 @@ inline void Vector::NewMemoryAndSize(const Memory<double> &mem, int s,
 {
    data.Delete();
    size = s;
-   data = mem;
-   if (!own_mem) { data.ClearOwnerFlags(); }
+   if (own_mem)
+   {
+      data = mem;
+   }
+   else
+   {
+      data.MakeAlias(mem, 0, s);
+   }
 }
 
 inline void Vector::MakeRef(Vector &base, int offset, int s)

makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.
 
 # The current MFEM version as an integer, see also `CMakeLists.txt`.
-MFEM_VERSION = 40201
+MFEM_VERSION = 40301
 MFEM_VERSION_STRING = $(shell printf "%06d" $(MFEM_VERSION) | \
   sed -e 's/^0*\(.*.\)\(..\)\(..\)$$/\1.\2.\3/' -e 's/\.0/./g' -e 's/\.0$$//')
 
@@ -119,7 +119,7 @@ $(if $(word 2,$(SRC)),$(error Spaces in SRC = "$(SRC)" are not supported))
 MFEM_GIT_STRING = $(shell [ -d $(MFEM_DIR)/.git ] && git -C $(MFEM_DIR) \
    describe --all --long --abbrev=40 --dirty --always 2> /dev/null)
 
-EXAMPLE_SUBDIRS = amgx ginkgo hiop petsc pumi sundials superlu
+EXAMPLE_SUBDIRS = amgx ginkgo hiop petsc pumi sundials epic superlu
 EXAMPLE_DIRS := examples $(addprefix examples/,$(EXAMPLE_SUBDIRS))
 EXAMPLE_TEST_DIRS := examples
 
@@ -209,27 +209,28 @@ CXXFLAGS ?= $(OPTIM_FLAGS)
 
 # MPI configuration
 ifneq ($(MFEM_USE_MPI),YES)
-   MFEM_HOST_CXX = $(CXX)
+   HOST_CXX = $(CXX)
    PKGS_NEED_MPI = SUPERLU MUMPS STRUMPACK PETSC PUMI SLEPC MKL_CPARDISO
    $(foreach mpidep,$(PKGS_NEED_MPI),$(if $(MFEM_USE_$(mpidep):NO=),\
      $(warning *** [MPI is OFF] setting MFEM_USE_$(mpidep) = NO)\
      $(eval override MFEM_USE_$(mpidep)=NO),))
 else
-   MFEM_HOST_CXX = $(MPICXX)
+   HOST_CXX = $(MPICXX)
    INCFLAGS += $(HYPRE_OPT)
    ALL_LIBS += $(HYPRE_LIB)
 endif
 
 # Default configuration
 ifeq ($(MFEM_USE_CUDA)$(MFEM_USE_HIP),NONO)
-   MFEM_CXX ?= $(MFEM_HOST_CXX)
-   MFEM_HOST_CXX := $(MFEM_CXX)
+   MFEM_CXX ?= $(HOST_CXX)
+   MFEM_HOST_CXX ?= $(MFEM_CXX)
    XCOMPILER = $(CXX_XCOMPILER)
    XLINKER   = $(CXX_XLINKER)
 endif
 
 ifeq ($(MFEM_USE_CUDA),YES)
    MFEM_CXX ?= $(CUDA_CXX)
+   MFEM_HOST_CXX ?= $(HOST_CXX)
    CXXFLAGS += $(CUDA_FLAGS) -ccbin $(MFEM_HOST_CXX)
    XCOMPILER = $(CUDA_XCOMPILER)
    XLINKER   = $(CUDA_XLINKER)
@@ -247,7 +248,7 @@ ifeq ($(MFEM_USE_HIP),YES)
       ALL_LIBS += $(MPI_LIB)
    endif
    MFEM_CXX ?= $(HIP_CXX)
-   MFEM_HOST_CXX := $(MFEM_CXX)
+   MFEM_HOST_CXX ?= $(MFEM_CXX)
    CXXFLAGS += $(HIP_FLAGS)
    XLINKER   = $(HIP_XLINKER)
    XCOMPILER = $(HIP_XCOMPILER)
@@ -271,7 +272,7 @@ ifeq ($(MFEM_USE_LEGACY_OPENMP),YES)
 endif
 
 # List of MFEM dependencies, that require the *_LIB variable to be non-empty
-MFEM_REQ_LIB_DEPS = SUPERLU MUMPS METIS CONDUIT SIDRE LAPACK SUNDIALS MESQUITE\
+MFEM_REQ_LIB_DEPS = SUPERLU MUMPS METIS FMS CONDUIT SIDRE LAPACK SUNDIALS EPIC MESQUITE\
  SUITESPARSE STRUMPACK GINKGO GNUTLS NETCDF PETSC SLEPC MPFR PUMI HIOP GSLIB\
  OCCA CEED RAJA UMPIRE MKL_CPARDISO AMGX CALIPER
 
@@ -332,14 +333,14 @@ endif
 MFEM_DEFINES = MFEM_VERSION MFEM_VERSION_STRING MFEM_GIT_STRING MFEM_USE_MPI\
  MFEM_USE_METIS MFEM_USE_METIS_5 MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB\
  MFEM_USE_LIBUNWIND MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP\
- MFEM_USE_LEGACY_OPENMP MFEM_USE_MEMALLOC MFEM_TIMER_TYPE MFEM_USE_SUNDIALS\
+ MFEM_USE_LEGACY_OPENMP MFEM_USE_MEMALLOC MFEM_TIMER_TYPE MFEM_USE_SUNDIALS MFEM_USE_EPIC\
  MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE MFEM_USE_GINKGO MFEM_USE_SUPERLU\
  MFEM_USE_STRUMPACK MFEM_USE_GNUTLS MFEM_USE_NETCDF MFEM_USE_PETSC\
- MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI\
- MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP MFEM_USE_OCCA\
- MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2\
- MFEM_USE_MKL_CPARDISO MFEM_USE_AMGX MFEM_USE_MUMPS MFEM_USE_CALIPER\
- MFEM_SOURCE_DIR MFEM_INSTALL_DIR
+ MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS MFEM_USE_CONDUIT\
+ MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA MFEM_USE_HIP\
+ MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD\
+ MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO MFEM_USE_AMGX MFEM_USE_MUMPS\
+ MFEM_USE_CALIPER MFEM_SOURCE_DIR MFEM_INSTALL_DIR
 
 # List of makefile variables that will be written to config.mk:
 MFEM_CONFIG_VARS = MFEM_CXX MFEM_HOST_CXX MFEM_CPPFLAGS MFEM_CXXFLAGS\
@@ -519,6 +520,17 @@ test test-noclean:
 	   if [ 0 -ne $${ERR} ]; then echo "Some tests failed."; exit 1; \
 	   else echo "All tests passed."; fi
 
+.PHONY: test-miniapps
+test-miniapps:
+	@echo "Building all miniapps ..."
+	@$(MAKE) $(MAKEOVERRIDES_SAVE) miniapps
+	@ERR=0; for dir in $(MINIAPP_TEST_DIRS); do \
+	   echo "Running tests in $${dir} ..."; \
+	   if ! $(MAKE) -j1 -C $(BLD)$${dir} test; then \
+	   ERR=1; fi; done; \
+	   if [ 0 -ne $${ERR} ]; then echo "Some miniapp tests failed."; \
+	   exit 1; else echo "All miniapp tests passed."; fi
+
 unittest: lib
 	$(MAKE) -C $(BLD)tests/unit test
 
@@ -638,6 +650,7 @@ status info:
 	$(info MFEM_USE_MEMALLOC      = $(MFEM_USE_MEMALLOC))
 	$(info MFEM_TIMER_TYPE        = $(MFEM_TIMER_TYPE))
 	$(info MFEM_USE_SUNDIALS      = $(MFEM_USE_SUNDIALS))
+	$(info MFEM_USE_EPIC          = $(MFEM_USE_EPIC))
 	$(info MFEM_USE_MESQUITE      = $(MFEM_USE_MESQUITE))
 	$(info MFEM_USE_SUITESPARSE   = $(MFEM_USE_SUITESPARSE))
 	$(info MFEM_USE_SUPERLU       = $(MFEM_USE_SUPERLU))
@@ -651,6 +664,7 @@ status info:
 	$(info MFEM_USE_SLEPC         = $(MFEM_USE_SLEPC))
 	$(info MFEM_USE_MPFR          = $(MFEM_USE_MPFR))
 	$(info MFEM_USE_SIDRE         = $(MFEM_USE_SIDRE))
+	$(info MFEM_USE_FMS           = $(MFEM_USE_FMS))
 	$(info MFEM_USE_CONDUIT       = $(MFEM_USE_CONDUIT))
 	$(info MFEM_USE_PUMI          = $(MFEM_USE_PUMI))
 	$(info MFEM_USE_HIOP          = $(MFEM_USE_HIOP))

mesh/mesh.cpp

@@ -599,6 +599,8 @@ void Mesh::GetEdgeTransformation(int EdgeNo, IsoparametricTransformation *EdTr)
    else
    {
       const FiniteElement *edge_el = Nodes->FESpace()->GetEdgeElement(EdgeNo);
+      Nodes->HostRead();
+      const GridFunction &nodes = *Nodes;
       if (edge_el)
       {
          Array<int> vdofs;
@@ -609,7 +611,7 @@ void Mesh::GetEdgeTransformation(int EdgeNo, IsoparametricTransformation *EdTr)
          {
             for (int j = 0; j < n; j++)
             {
-               pm(i, j) = (*Nodes)(vdofs[n*i+j]);
+               pm(i, j) = nodes(vdofs[n*i+j]);
             }
          }
          EdTr->SetFE(edge_el);
@@ -4108,7 +4110,11 @@ void Mesh::MakeRefined_(Mesh &orig_mesh, const Array<int> ref_factors,
    }
 
    MFEM_ASSERT(CheckElementOrientation(false) == 0, "");
-   MFEM_ASSERT(CheckBdrElementOrientation(false) == 0, "");
+
+   // The check below is disabled because is fails for parallel meshes with
+   // interior "boundary" element that, when such "boundary" element is between
+   // two elements on different processors.
+   // MFEM_ASSERT(CheckBdrElementOrientation(false) == 0, "");
 }
 
 Mesh Mesh::MakeSimplicial(const Mesh &orig_mesh)

mesh/mesh.hpp

@@ -811,11 +811,6 @@ public:
    MFEM_DEPRECATED
    Mesh(Mesh *orig_mesh, int ref_factor, int ref_type);
 
-   /// A version of the above constructor for non-uniform refinement.
-   /** The input array @a ref_factors contains one refinement factor per element
-       of the input mesh. */
-   Mesh(Mesh *orig_mesh, const Array<int> &ref_factors, int ref_type);
-
    /** This is similar to the mesh constructor with the same arguments, but here
        the current mesh is destroyed and another one created based on the data
        stream again given in MFEM, Netgen, or VTK format. If generate_edges = 0
@@ -877,17 +872,26 @@ public:
    long GetGlobalNE() const { return ReduceInt(NumOfElements); }
 
    /** @brief Return the mesh geometric factors corresponding to the given
-       integration rule. */
-   /** If the device MemoryType parameter @a d_mt is specified, then the
-       returned object will use that type unless it was previously allocated
-       with a different type. */
+       integration rule.
+
+       The IntegrationRule used with GetGeometricFactors needs to remain valid
+       until the internally stored GeometricFactors objects are destroyed (by
+       either calling Mesh::DeleteGeometricFactors or the Mesh destructor). If
+       the device MemoryType parameter @a d_mt is specified, then the returned
+       object will use that type unless it was previously allocated with a
+       different type. */
    const GeometricFactors* GetGeometricFactors(
       const IntegrationRule& ir,
       const int flags,
       MemoryType d_mt = MemoryType::DEFAULT);
 
    /** @brief Return the mesh geometric factors for the faces corresponding
-        to the given integration rule. */
+       to the given integration rule.
+
+       The IntegrationRule used with GetFaceGeometricFactors needs to remain
+       valid until the internally stored FaceGeometricFactors objects are
+       destroyed (by either calling Mesh::DeleteGeometricFactors or the Mesh
+       destructor). */
    const FaceGeometricFactors* GetFaceGeometricFactors(const IntegrationRule& ir,
                                                        const int flags,
                                                        FaceType type);

miniapps/gslib/schwarz_ex1p.cpp

@@ -327,7 +327,7 @@ int main(int argc, char *argv[])
       {
          bnd(i+d*number_boundary) = vxyz(idx + d*number_true);
       }
-      n      colorv[i] = (unsigned int)color;
+      colorv[i] = (unsigned int)color;
    }
    Vector interp_vals1(number_boundary);
    finder.Interpolate(bnd, colorv, x, interp_vals1);

miniapps/meshing/polar-nc.cpp

@@ -557,4 +557,3 @@ int main(int argc, char *argv[])
 
    return EXIT_SUCCESS;
 }
-

miniapps/mtop/paramnonlinearform.cpp

@@ -339,7 +339,7 @@ double ParametricBNLForm::GetEnergyBlocked(const BlockVector &bx,
 
 void ParametricBNLForm::SetStateFields(const Vector &xv) const
 {
-   BlockVector bx(xv.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(xv), block_trueOffsets);
    if (needs_prolongation)
    {
       for (int s = 0; s < fes.Size(); s++)
@@ -355,7 +355,7 @@ void ParametricBNLForm::SetStateFields(const Vector &xv) const
 
 void ParametricBNLForm::SetAdjointFields(const Vector &av) const
 {
-   BlockVector bx(av.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(av), block_trueOffsets);
    if (needs_prolongation)
    {
       for (int s = 0; s < fes.Size(); s++)
@@ -372,7 +372,7 @@ void ParametricBNLForm::SetAdjointFields(const Vector &av) const
 
 void ParametricBNLForm::SetParamFields(const Vector &dv) const
 {
-   BlockVector bx(dv.GetData(), paramblock_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(dv), paramblock_trueOffsets);
    if (prmneeds_prolongation)
    {
       for (int s = 0; s < paramfes.Size(); s++)
@@ -388,7 +388,7 @@ void ParametricBNLForm::SetParamFields(const Vector &dv) const
 
 double ParametricBNLForm::GetEnergy(const Vector &x) const
 {
-   xs.Update(x.GetData(),block_offsets);
+   xs.Update(const_cast<Vector&>(x),block_offsets);
    return GetEnergyBlocked(xs,xdv);
 }
 
@@ -898,8 +898,8 @@ const
 
 void ParametricBNLForm::ParamMult(const Vector &x, Vector &y) const
 {
-   BlockVector bx(x.GetData(), paramblock_trueOffsets);
-   BlockVector by(y.GetData(), paramblock_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), paramblock_trueOffsets);
+   BlockVector by(y, paramblock_trueOffsets);
 
    const BlockVector &pbx = ParamProlongate(bx);
 
@@ -909,8 +909,8 @@ void ParametricBNLForm::ParamMult(const Vector &x, Vector &y) const
    }
    BlockVector &pby = prmneeds_prolongation ? prmaux2 : by;
 
-   xs.Update(pbx.GetData(), paramblock_offsets);
-   ys.Update(pby.GetData(), paramblock_offsets);
+   xs.Update(const_cast<BlockVector&>(pbx), paramblock_offsets);
+   ys.Update(pby, paramblock_offsets);
 
    MultParamBlocked(xsv,adv,xs,ys);
 
@@ -928,8 +928,8 @@ void ParametricBNLForm::ParamMult(const Vector &x, Vector &y) const
 void ParametricBNLForm::Mult(const Vector &x, Vector &y) const
 {
 
-   BlockVector bx(x.GetData(), block_trueOffsets);
-   BlockVector by(y.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
+   BlockVector by(y, block_trueOffsets);
 
    const BlockVector &pbx = Prolongate(bx);
 
@@ -939,8 +939,8 @@ void ParametricBNLForm::Mult(const Vector &x, Vector &y) const
    }
    BlockVector &pby = needs_prolongation ? aux2 : by;
 
-   xs.Update(pbx.GetData(), block_offsets);
-   ys.Update(pby.GetData(), block_offsets);
+   xs.Update(const_cast<BlockVector&>(pbx), block_offsets);
+   ys.Update(pby, block_offsets);
    MultBlocked(xs,xdv,ys);
 
    for (int s = 0; s < fes.Size(); s++)
@@ -1198,7 +1198,7 @@ void ParametricBNLForm::ComputeGradientBlocked(const BlockVector &bx,
 
 BlockOperator& ParametricBNLForm::GetGradient(const Vector &x) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
    const BlockVector &pbx = Prolongate(bx);
 
    ComputeGradientBlocked(pbx, xdv);

miniapps/mtop/parheat.cpp

@@ -203,11 +203,11 @@ int main(int argc, char *argv[])
    nf->SetParamFields(prmbv); //set the density
 
    // Compute the stiffness/tangent matrix for density prmbv=0.5.
-   mfem::BlockOperator& A=nf->GetGradient(solbv);
+   mfem::BlockOperator *A = &nf->GetGradient(solbv);
    mfem::HypreBoomerAMG* prec=new mfem::HypreBoomerAMG();
    prec->SetPrintLevel(print_level);
    // Use only block (0,0) as in this case we have a single field.
-   prec->SetOperator(A.GetBlock(0,0));
+   prec->SetOperator(A->GetBlock(0,0));
 
    // Construct block preconditioner for the BNLForm.
    mfem::BlockDiagonalPreconditioner *blpr = new mfem::BlockDiagonalPreconditioner(
@@ -222,7 +222,7 @@ int main(int argc, char *argv[])
    gmres->SetMaxIter(100);
    gmres->SetPrintLevel(print_level);
    gmres->SetPreconditioner(*blpr);
-   gmres->SetOperator(A);
+   gmres->SetOperator(*A);
 
 
    // Solve the problem.
@@ -319,10 +319,10 @@ int main(int argc, char *argv[])
          // Solve the physics.
          solbv=0.0;
          nf->Mult(solbv,resbv); resbv.Neg(); //compute RHS
-         A=nf->GetGradient(solbv);
+         A = &nf->GetGradient(solbv);
          prec->SetPrintLevel(0);
-         prec->SetOperator(A.GetBlock(0,0));
-         gmres->SetOperator(A);
+         prec->SetOperator(A->GetBlock(0,0));
+         gmres->SetOperator(*A);
          gmres->SetPrintLevel(0);
          gmres->Mult(resbv,solbv);
          // Compute the objective.

miniapps/mtop/pparamnonlinearform.cpp

@@ -124,7 +124,7 @@ void ParParametricBNLForm::SetParamEssentialBC(const
 
 double ParParametricBNLForm::GetEnergy(const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   xs_true.Update(const_cast<Vector&>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s = 0; s < fes.Size(); ++s)
@@ -143,8 +143,8 @@ double ParParametricBNLForm::GetEnergy(const Vector &x) const
 
 void ParParametricBNLForm::Mult(const Vector &x, Vector &y) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
-   ys_true.Update(y.GetData(), block_trueOffsets);
+   xs_true.Update(const_cast<Vector&>(x), block_trueOffsets);
+   ys_true.Update(y, block_trueOffsets);
    xs.Update(block_offsets);
    ys.Update(block_offsets);
 
@@ -173,8 +173,8 @@ void ParParametricBNLForm::Mult(const Vector &x, Vector &y) const
 /// Block T-Vector to Block T-Vector
 void ParParametricBNLForm::ParamMult(const Vector &x, Vector &y) const
 {
-   xs_true.Update(x.GetData(), paramblock_trueOffsets);
-   ys_true.Update(y.GetData(), paramblock_trueOffsets);
+   xs_true.Update(const_cast<Vector&>(x), paramblock_trueOffsets);
+   ys_true.Update(y, paramblock_trueOffsets);
    prmxs.Update(paramblock_offsets);
    prmys.Update(paramblock_offsets);
 
@@ -205,7 +205,7 @@ void ParParametricBNLForm::ParamMult(const Vector &x, Vector &y) const
 const BlockOperator & ParParametricBNLForm::GetLocalGradient(
    const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   xs_true.Update(const_cast<Vector&>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s=0; s<fes.Size(); ++s)
@@ -325,7 +325,7 @@ ParParametricBNLForm::~ParParametricBNLForm()
 
 void ParParametricBNLForm::SetStateFields(const Vector &xv) const
 {
-   xs_true.Update(xv.GetData(), block_trueOffsets);
+   xs_true.Update(const_cast<Vector&>(xv), block_trueOffsets);
    xsv.Update(block_offsets);
    for (int s=0; s<fes.Size(); ++s)
    {
@@ -337,7 +337,7 @@ void ParParametricBNLForm::SetStateFields(const Vector &xv) const
 
 void ParParametricBNLForm::SetAdjointFields(const Vector &av) const
 {
-   xs_true.Update(av.GetData(), block_trueOffsets);
+   xs_true.Update(const_cast<Vector&>(av), block_trueOffsets);
    adv.Update(block_offsets);
    for (int s=0; s<fes.Size(); ++s)
    {
@@ -348,7 +348,7 @@ void ParParametricBNLForm::SetAdjointFields(const Vector &av) const
 
 void ParParametricBNLForm::SetParamFields(const Vector &dv) const
 {
-   xs_true.Update(dv.GetData(),paramblock_trueOffsets);
+   xs_true.Update(const_cast<Vector&>(dv),paramblock_trueOffsets);
    xdv.Update(paramblock_offsets);
    for (int s=0; s<paramfes.Size(); ++s)
    {

miniapps/navier/navier_cht.cpp

@@ -100,8 +100,8 @@ protected:
    HypreParMatrix *T; // T = M + dt K
    double current_dt;
 
-   mutable CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
-   HypreSmoother M_prec; // Preconditioner for the mass matrix M
+   mutable CGSolver M_solver; // Krylov solver for inverting the mass matrix M
+   HypreSmoother M_prec;      // Preconditioner for the mass matrix M
 
    CGSolver T_solver;    // Implicit solver for T = M + dt K
    HypreSmoother T_prec; // Preconditioner for the implicit solver
@@ -421,10 +421,10 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
    Array<int> ess_bdr(f.GetParMesh()->bdr_attributes.Max());
    // Dirichlet boundary condition on inlet and isothermal section of wall.
    ess_bdr = 0;
-   ess_bdr[0] = 1; //inlet
-   ess_bdr[1] = 1; //homogeneous isothermal section of bottom wall
-   ess_bdr[2] = 0; //top wall
-   ess_bdr[3] = 0; //inhomogeneous isothermal section of bottom wall
+   ess_bdr[0] = 1; // inlet
+   ess_bdr[1] = 1; // homogeneous isothermal section of bottom wall
+   ess_bdr[2] = 0; // top wall
+   ess_bdr[3] = 0; // inhomogeneous isothermal section of bottom wall
    f.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
    M = new ParBilinearForm(&fespace);

miniapps/navier/navier_solver.cpp

@@ -565,10 +565,11 @@ void NavierSolver::Step(double &time, double dt, int cur_step, bool provisional)
       un_filtered_gf.ProjectGridFunction(un_NM1_gf);
       const auto d_un_filtered_gf = un_filtered_gf.Read();
       auto d_un_gf = un_gf.ReadWrite();
+      const auto filter_alpha_ = filter_alpha;
       MFEM_FORALL(i,
                   un_gf.Size(),
-                  d_un_gf[i] = (1.0 - filter_alpha) * d_un_gf[i]
-                               + filter_alpha * d_un_filtered_gf[i];);
+                  d_un_gf[i] = (1.0 - filter_alpha_) * d_un_gf[i]
+                               + filter_alpha_ * d_un_filtered_gf[i];);
    }
 
    sw_step.Stop();
@@ -1019,7 +1020,7 @@ void NavierSolver::AddAccelTerm(VecFuncT *f, Array<int> &attr)
 
 void NavierSolver::SetTimeIntegrationCoefficients(int step)
 {
-   // Maxmium BDF order to use at current time step
+   // Maximum BDF order to use at current time step
    // step + 1 <= order <= max_bdf_order
    int bdf_order = std::min(step + 1, max_bdf_order);
 

miniapps/navier/navier_tgv.cpp

@@ -327,7 +327,7 @@ int main(int argc, char *argv[])
    double ke = kin_energy.ComputeKineticEnergy(*u_gf);
 
    std::string fname = "tgv_out_p_" + std::to_string(ctx.order) + ".txt";
-   FILE *f;
+   FILE *f = NULL;
 
    if (mpi.Root())
    {

miniapps/navier/ortho_solver.cpp

@@ -46,6 +46,8 @@ void OrthoSolver::Orthogonalize(const Vector &v, Vector &v_ortho) const
 
    double ratio = global_sum / static_cast<double>(global_size);
    v_ortho.SetSize(v.Size());
+   v.HostRead();
+   v_ortho.HostWrite();
    for (int i = 0; i < v_ortho.Size(); ++i)
    {
       v_ortho(i) = v(i) - ratio;