Fix another makefile bug

.appveyor.yml

@@ -26,25 +26,24 @@ install:
 - cd ..
 
 # Install hypre
-- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
-- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
-- cd hypre-2-10-0b
-- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz'
+- 7z x hypre-2.10.0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2.10.0b
+- cmake -Hsrc -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+# - cmake -Hsrc -Bbuild -DCMAKE_BUILD_TYPE=Release -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
 - cd ..
 
 # MFEM
 before_build:
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 
 build_script:
 - cmake --build build_parallel
 - cmake --build build_serial
-- cmake --build build_serial --target exec
 
 after_build:
 # - cmake --build build_parallel --target check
-- cmake --build build_serial --target RUN_TESTS
-
+- cmake --build build_serial --target check

.gitignore

@@ -82,9 +82,9 @@ examples/ex20.dat
 examples/ex20p_?????.dat
 examples/gnuplot_ex20.inp
 examples/gnuplot_ex20p.inp
-examples/ex21*.mesh
-examples/ex21*.sol
-examples/ex21p_*.*
+examples/ex22*.mesh
+examples/ex22*.sol
+examples/ex22p_*.*
 
 examples/sundials/ex9
 examples/sundials/ex1[06]
@@ -169,7 +169,6 @@ miniapps/performance/sol.*
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
-miniapps/tools/lor-transfer
 
 miniapps/nurbs/ex1
 miniapps/nurbs/ex1p
@@ -183,6 +182,3 @@ miniapps/nurbs/Example1*
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
-
-# VPATH builds
-build-*/*

.travis.yml

@@ -205,7 +205,6 @@ install:
         else
             echo "Reusing cached hypre-2.10.0b/";
         fi;
-        ln -s hypre-2.10.0b hypre;
      else
         echo "Serial build, not using hypre";
      fi

CHANGELOG

@@ -8,132 +8,22 @@
                                http://mfem.org
 
 
-Version 4.0.1 (development)
+Version 3.4.1 (development)
 ===========================
 
-Improved GPU support
---------------------
-- Added initial support for AMD GPUs based on HIP: a C++ runtime API and kernel
-  language that can run on both AMD and NVIDIA hardware. With this change, the
-  list of backends is: "occa-cuda", "raja-cuda", "cuda", "hip", "occa-omp",
-  "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".
-
-- Improved RAJA backend and multi-GPU MPI communications.
-
-Discretization improvements
----------------------------
-- Added support for non-conforming prism AMR, including coarsening and parallel
-  load balancing. Anisotropic prism refinement is only available in the serial
-  version at the moment.
-
-Meshing improvements
---------------------
-- The TMOP mesh optimization algorithms were extended to support r-adaptivity.
-  Target matrices can now be constructed either via a given analytical function
-  (e.g. spatial dependence of size, aspect ratio, etc., for each element) or via
-  a (Par)GridFunction specified on the original mesh.
-
-- The TMOP mesh optimization algorithms have been improved to support AMR meshes.
-
-- Added support for creating refined versions of periodic meshes, making use of
-  the new L2ElementRestriction class. This class also allows for computing
-  geometric factors on periodic meshes using partial assembly.
-
-- Improved element numbering after uniform mesh refinement.
-
-New and updated examples and miniapps
--------------------------------------
-- The mesh-optimizer and pmesh-optimizer miniapps have been updated to
-  demonstrate the new r-adaptivity capabilities of TMOP.
-
-- The (p)mesh-optimizer miniapp has been updated to demonstrate mesh
-  optimization for an AMR mesh.
-
-Miscellaneous
--------------
-- Upgraded the SUNDIALS interface to utilize SUNDIALS version 5.0.  This
-  necessitated a complete rework of the interface and requires changes at
-  the application level.  Example usage of this new interface can be found
-  in the examples/sundials directory.
-
-
-Version 4.0, released on May 24, 2019
-=====================================
-
-Unlike previous MFEM releases, this version requires a C++11 compiler.
-
-GPU support
------------
-- Added initial support for hardware devices, such as GPUs, and programming
-  models, such as CUDA, OCCA, RAJA and OpenMP.
-
-- The GPU/device support is based on MFEM's new backends and kernels working
-  seamlessly with a new lightweight device/host memory manager. The kernels can
-  be implemented either in OCCA, or as a simple wrapper around for-loops, which
-  can then be dispatched to RAJA and native backends. See the files forall.hpp
-  and mem_manager.hpp in the general/ directory for more details.
-
-- Several of the MFEM example codes (ex1, ex1p, ex6, and ex6p) can now take
-  advantage of GPU acceleration with the backend selectable at runtime. Many of
-  the linear algebra and finite element operations (e.g. partially assembled
-  bilinear forms) have been extended to take advantage of kernel acceleration by
-  simply replacing loops with the MFEM_FORALL() macro.
-
-- In addition to native CUDA kernels, the library currently supports OCCA, RAJA
-  and OpenMP kernels, which could be mixed and matched in different parts of the
-  same application. We plan on adding support for more programming models and
-  devices in the future, without the need for significant modifications in user
-  code. The list of current backends is: "occa-cuda", "raja-cuda", "cuda",
-  "occa-omp", "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".
-
-- GPU-related limitations:
-  * Hypre preconditioners are not yet available in GPU mode, and in particular
-    hypre must be built in CPU mode.
-  * Only constant coefficients are currently supported on GPUs.
-  * Optimized element assembly, and matrix-free bilinear forms are not
-    implemented yet. Element batching is currently ignored.
-  * In device mode, full assembly is performed on the host (but the matvec
-    action is performed on the device).
-  * Partial assembly kernels are not implemented yet for simplices.
-
-Discretization improvements
----------------------------
-- Partial assembled finite element operators are now available in the core
-  library, based on the new classes PABilinearFormExtension, ElementRestriction,
-  DofToQuad and GeometricFactors (associated with the classes BilinearForm,
-  FiniteElementSpace, FiniteElement and Mesh, respectively). The kernels for
-  partial assembled Setup/Assembly and Action/Mult are implemented in the
-  BilinearFormIntegrator methods AssemblePA and AddMultPA.
-
-- Added support for a general "low-order refined"-to-"high-order" transfer of
-  GridFunction data from a "low-order refined" (LOR) space defined on a refined
-  mesh to a "high-order" (HO) finite element space defined on a coarse mesh. See
-  the new classes InterpolationGridTransfer and L2ProjectionGridTransfer and the
-  new LOR Transfer miniapp: miniapps/tools/lor-transfer.cpp.
-
-- Added element flux, and flux energy computation in class ElasticityIntegrator,
-  allowing for the use of Zienkiewicz-Zhu type error estimators with the
-  integrator. For an illustration of this addition, see the new Example 21.
-
-- Added support for derefinement of vector (RT + ND) spaces.
-
-- Added a variety of coefficients which are sums or products of existing
-  coefficients as well as grid function coefficients which return the
-  divergence, gradient, or curl of their GridFunctions.
-
 Support for wedge elements and meshes with mixed element types
 --------------------------------------------------------------
-- Added support for wedge-shaped mesh elements of arbitrary order (with Geometry
+- Added support for wedge shaped mesh elements of arbitrary order (with Geometry
   type PRISM) which have two triangular faces and three quadrilateral faces.
   Several examples of such meshes can be found in the data/ directory.
 
+- Added H1 and L2 finite elements of arbitrary order for Wedge elements.
+
 - Added support for mixed meshes containing triangles and quadrilaterals in 2D
   or tetrahedra, wedges, and hexahedra in 3D. This includes support for uniform
   refinement of such meshes. Several examples of such meshes can be found in the
   data/ directory.
 
-- Added H1 and L2 finite elements of arbitrary order for Wedge elements.
-
 - Added support for reading and writing linear and quadratic meshes containing
   wedge elements in VTK mesh format. Several examples of such meshes can be
   found in the data/ directory.
@@ -149,28 +39,39 @@ Other meshing improvements
   follows precisely the paper:
 
      D. Arnold, A. Mukherjee, and L. Pouly, "Locally Adapted Tetrahedral Meshes
-     Using Bisection", SIAM J. Sci. Comput. 22 (2000), 431–448.
+     Using Bisection", SIAM J. Sci. Comput., 22(2), 431–448.
 
   This guarantees that the shape regularity of the elements will be preserved
   under refinement.
 
-- The TMOP mesh optimization algorithms were extended to support user-defined
-  space-dependent limiting terms. Improved the TMOP objective functions by more
-  accurate normalization of the different terms.
-
 - Added support for parallel communication groups on non-conforming meshes.
 
-- Improved parallel partitioning of non-conforming meshes. If the coarse mesh
-  elements are ordered as a sequence of face-neighbors, the parallel partitions
-  are now guaranteed to be continuous. To that end, inline quadrilateral and
-  hexahedral meshes are now by default ordered along a space-filling curve.
-
 - A boundary in a NURBS mesh can now be connected with another boundary. Such a
   periodic NURBS mesh is a simple way to impose periodic boundary conditions.
 
 - Added support for reading linear and quadratic 2D quadrilateral and triangular
   Cubit meshes.
 
+- The TMOP mesh optimization algorithms were extended to support user-defined
+  space-dependent limiting terms. Improved the TMOP objective functions by
+  more accurate normalization of the different terms.
+  
+Discretization improvements
+---------------------------
+- Added support for derefinement of vector (RT + ND) spaces.
+
+- Added element flux, and flux energy computation in class ElasticityIntegrator,
+  allowing for the use of Zienkiewicz-Zhu type error estimators with the
+  integrator. For an illustration of this addition, see the new Example 22.
+
+- Added a variety of coefficients which are sums or products of existing
+  coefficients as well as grid function coefficients which return the
+  divergence, gradient, or curl of their GridFunctions.
+
+New and improved solvers and preconditioners
+--------------------------------------------
+- Added support for parallel ILU preconditioning via hypre's Euclid solver.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new meshing miniapp, Toroid, which can produce a variety of torus
@@ -179,43 +80,21 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Extruder, that demonstrates the capability to
   produce 3D meshes by extruding 2D meshes.
 
-- Added a simple miniapp, LOR Transfer, for visualizing the actions of the
-  transfer operators between a high-order and a low-order refined spaces.
-
 - Added a new example, Example 20/20p, that solves a system of 1D ODEs derived
   from a Hamiltonian. The example demonstrates the use of the variable order,
   symplectic integration algorithm implemented in class SIAVSolver.
 
-- Added a new example, Example 21/21p, that illustrates the use of AMR to solve
+- Added a new example, Example 22/22p, that illustrates the use of AMR to solve
   a linear elasticity problem. This is an extension of Example 2/2p.
 
-New and improved solvers and preconditioners
---------------------------------------------
-- Added support for parallel ILU preconditioning via hypre's Euclid solver.
-
-- Added support for STRUMPACK v3 with a small API change in the class
-  STRUMPACKSolver, see "API changes" below.
-
 Miscellaneous
 -------------
-- Added unit tests based on the Catch++ library in the test/ directory.
-
-- Renamed the option MFEM_USE_OPENMP to MFEM_USE_LEGACY_OPENMP. This legacy
-  option is deprecated and planned for removal in a future release. The original
-  option name, MFEM_USE_OPENMP, is now used to enable the new OpenMP backends in
-  the new kernels.
-
-- In SparseMatrix added the option to perform MultTranspose() by matvec with
-  computed and stored transpose matrix. This is required for deterministic
-  results when using devices such as CUDA and OpenMP.
+- Added unit tests based on the Catch++ library.
 
 - Altered the way FGMRES counts its iterations so that it matches GMRES.
 
 - Various other simplifications, extensions, and bugfixes in the code.
 
-- Construct abstract parallel rectangular truedof-to-truedof operators via
-  Operator::FormDiscreteOperator().
-
 API changes
 -----------
 - In multiple places, use Geometry::Type instead of int, where appropriate.
@@ -236,8 +115,6 @@ API changes
 - Removed the virtual method Element::GetRefinementFlag, it is only used by the
   derived class Tetrahedron.
 - Added new methods: Array::CopyTo, Tetrahedron::Init.
-- In class STRUMPACKSolver, the method SetMC64Job() was replaced by the new
-  methods: DisableMatching(), EnableMatching(), and EnableParallelMatching().
 
 
 Version 3.4, released on May 29, 2018

CMakeLists.txt

@@ -13,11 +13,6 @@ cmake_minimum_required(VERSION 2.8.11)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
-# Require C++11 and disable compiler-specific extensions
-set(CMAKE_CXX_STANDARD 11)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS OFF)
-
 # Load user settings before the defaults - this way the defaults will not
 # overwrite the user set options. If the user has not set all options, we still
 # have the defaults.
@@ -50,7 +45,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.0.1)
+set(${PROJECT_NAME}_VERSION 3.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -86,13 +81,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 
 # Enable languages.
 enable_language(CXX)
-if (MFEM_USE_CUDA)
-   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
-   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
-   enable_language(CUDA)
-   message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
-endif()
-
 if (XSDK_ENABLE_C)
    enable_language(C)
 endif()
@@ -182,11 +170,12 @@ if (MFEM_USE_LAPACK)
 endif()
 
 # OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
+if (MFEM_USE_OPENMP)
+  if (MFEM_THREAD_SAFE)
+    find_package(OpenMP REQUIRED)
+  else()
+    message(FATAL_ERROR " *** MFEM_USE_OPENMP requires MFEM_THREAD_SAFE=ON.")
   endif()
-  find_package(OpenMP REQUIRED)
 endif()
 
 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -254,7 +243,7 @@ endif()
 
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
-  find_package(Axom REQUIRED Axom)
+  find_package(Axom REQUIRED Sidre SLIC axom_utils)
 endif()
 
 # PUMI
@@ -273,31 +262,6 @@ if (MFEM_USE_PUMI)
   endif()
 endif()
 
-# CUDA
-if (MFEM_USE_CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-   set(CMAKE_CUDA_EXTENSIONS OFF)
-   set(CMAKE_CUDA_FLAGS "-arch=${CUDA_ARCH} --expt-extended-lambda"
-       CACHE STRING "CUDA flags set for MFEM" FORCE)
-   if (MFEM_USE_MPI)
-      set(CUDA_CCBIN_COMPILER ${MPI_CXX_COMPILER})
-   else()
-      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
-   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
-endif()
-
-# OCCA
-if (MFEM_USE_OCCA)
-   find_package(OCCA REQUIRED)
-endif()
-
-# RAJA
-if (MFEM_USE_RAJA)
-   find_package(RAJA REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -323,7 +287,7 @@ endif()
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
     MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
-    POSIXCLOCKS MFEMBacktrace ZLIB OCCA RAJA)
+    POSIXCLOCKS MFEMBacktrace ZLIB)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -359,13 +323,6 @@ set(MFEM_SOURCE_DIRS general linalg mesh fem)
 foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
   add_subdirectory(${DIR})
 endforeach()
-
-if (MFEM_USE_CUDA)
-   foreach(file IN LISTS SOURCES)
-      set_property(SOURCE ${file} PROPERTY LANGUAGE CUDA)
-   endforeach()
-endif()
-
 add_subdirectory(config)
 set(MASTER_HEADERS
   ${PROJECT_SOURCE_DIR}/mfem.hpp
@@ -376,11 +333,6 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON CACHE BOOL "")
 set(CMAKE_INSTALL_RPATH "${_lib_path}" CACHE PATH "")
 set(CMAKE_INSTALL_NAME_DIR "${_lib_path}" CACHE PATH "")
 
-set(MFEM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH
-    "The MFEM source directory" FORCE)
-set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
-    "The MFEM install directory" FORCE)
-
 # Declaring the library
 add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
@@ -395,11 +347,11 @@ endif()
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 
-# If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
-# inside the build directory.
+# If building out-of-source, define MFEM_BUILD_DIR to point to the build
+# directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
   target_compile_definitions(mfem PRIVATE
-    "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
+    "MFEM_BUILD_DIR=${PROJECT_BINARY_DIR}")
 endif()
 
 # Generate configuration file in the build directory: config/_config.hpp.
@@ -415,7 +367,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
       "Writing substitute header --> \"${Header}\"")
     file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
 "// Auto-generated file.
-#define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
+#define MFEM_BUILD_DIR ${PROJECT_BINARY_DIR}
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
     # This version will be installed in the top include directory:
@@ -478,12 +430,12 @@ endif()
 # Add 'check' target - quick test
 if (NOT MFEM_USE_MPI)
   add_custom_target(check
-    ${CMAKE_CTEST_COMMAND} -R \"^ex1_ser\" -C ${CMAKE_CFG_INTDIR}
+    ${CMAKE_CTEST_COMMAND} -R '^ex1_ser' -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
   add_dependencies(check ex1)
 else()
   add_custom_target(check
-    ${CMAKE_CTEST_COMMAND} -R \"^ex1p\" -C ${CMAKE_CFG_INTDIR}
+    ${CMAKE_CTEST_COMMAND} -R '^ex1p' -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
   add_dependencies(check ex1p)
 endif()
@@ -528,13 +480,6 @@ install(DIRECTORY ${MFEM_SOURCE_DIRS}
   DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
   FILES_MATCHING PATTERN "*.hpp")
 
-# Install the okl files
-if (MFEM_USE_OCCA)
-  install(DIRECTORY ${MFEM_SOURCE_DIRS}
-    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "*.okl")
-endif()
-
 # Install ${HEADERS}
 # ---
 # foreach (HDR ${HEADERS})

CONTRIBUTING.md

@@ -142,16 +142,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
   + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
   + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
 
-- GPU and multi-core CPU support is based on device kernels supporting different
-  backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
-  device/host memory manager.
-
-- The main device-relevant classes and sources are:
-  + [`Device`](http://mfem.github.io/doxygen/html/device_8hpp.html)
-  + [`MemoryManager`](http://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](http://mfem.github.io/doxygen/html/forall_8hpp.html) macro
-  + the [`cuda.hpp`](http://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](http://mfem.github.io/doxygen/html/occa_8hpp.html) files
-
 - The `general/` directory contains C++ classes that serve as utilities for
   communication, error handling, arrays, (Boolean) tables, timing, etc.
 

INSTALL

@@ -13,41 +13,14 @@ of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
 requires an MPI C++ compiler, as well as the following external libraries:
 
 - hypre (a library of high-performance preconditioners)
-  https://github.com/hypre-space/hypre
+  http://www.llnl.gov/CASC/hypre
 
 - METIS (a family of multilevel partitioning algorithms)
   http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
 
-The hypre dependency can be downloaded as a tarball from GitHub or from the
-project webpage https://www.llnl.gov/casc/hypre. For example, the 2.16.0 release
-of hypre is available at
-
-   https://github.com/hypre-space/hypre/archive/v2.16.0.tar.gz
-
 The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.
 
-MFEM also includes support for devices such as GPUs, and programming models such
-as CUDA, HIP, OCCA, OpenMP and RAJA.
-
-- Starting with version 4.0, MFEM requires a C++11 compiler
-
-- CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
-  https://developer.nvidia.com/cuda-toolkit
-
-- HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
-
-- OCCA support requires the OCCA library
-  https://libocca.org
-
-- OpenMP support requires a compiler implementing the OpenMP API
-  https://www.openmp.org
-
-- RAJA support requires installation of the RAJA performance portability layer
-  with (optionally) support for CUDA and OpenMP
-  https://github.com/LLNL/RAJA
-
 The library supports two build systems: one based on GNU make, and a second one
 based on CMake. Both build systems are described below. Some hints for building
 without GNU make or CMake can be found at the end of this file.
@@ -57,7 +30,7 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
+- Homebrew/Science, https://github.com/Homebrew/homebrew-science
 
 We also recommend downloading and building the MFEM-based GLVis visualization
 tool which can be used to visualize the meshes and solution in MFEM's examples
@@ -69,19 +42,11 @@ Serial build:
    make serial -j 4
 
 Parallel build:
-   (download hypre and METIS 4 from above URLs)
+   (download hypre 2.10.0b and METIS 4 from above URLs)
    (build METIS 4 in ../metis-4.0 relative to mfem/)
-   (build hypre in ../hypre relative to mfem/)
+   (build hypre 2.10.0b in ../hypre-2.10.0b relative to mfem/)
    make parallel -j 4
 
-CUDA build:
-   make cuda -j 4
-   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')
-
-HIP build:
-   make hip -j 4
-   (build for a specific AMD GPU chip: 'make hip -j 4 HIP_ARCH=gfx900')
-
 Example codes (serial/parallel, depending on the build):
    cd examples
    make -j 4
@@ -92,6 +57,7 @@ Build everything (library, examples and miniapps) with current configuration:
 Quick-check the build by running Example 1/1p (optional):
    make check
 
+
 Quick start with CMake
 ======================
 Serial build:
@@ -100,19 +66,13 @@ Serial build:
    make -j 4   (assuming "UNIX Makefiles" generator)
 
 Parallel build:
-   (download hypre and METIS 4 from above URLs)
+   (download hypre 2.10.0b and METIS 4 from above URLs)
    (build METIS 4 in ../metis-4.0 relative to mfem/)
-   (build hypre in ../hypre relative to mfem/)
+   (build hypre 2.10.0b in ../hypre-2.10.0b relative to mfem/)
    mkdir <mfem-build-dir> ; cd <mfem-build-dir>
    cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
    make -j 4
 
-CUDA build:
-   (this build requires CMake 3.8 or newer)
-   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_CUDA=YES
-   make -j 4
-
 Example codes (serial/parallel, depending on the build):
    make examples -j 4
 
@@ -168,18 +128,10 @@ Note that re-configuration is only needed to change the currently configured
 options. Several shortcut targets combining (re-)configuration and compilation
 are also defined:
 
-   make serial    -> Builds serial optimized version of the library
-   make parallel  -> Builds parallel optimized version of the library
-   make debug     -> Builds serial debug version of the library
-   make pdebug    -> Builds parallel debug version of the library
-   make cuda      -> Builds serial cuda optimized version of the library
-   make pcuda     -> Builds parallel cuda optimized version of the library
-   make cudebug   -> Builds serial cuda debug version of the library
-   make pcudebug  -> Builds parallel cuda debug version of the library
-   make hip       -> Builds serial hip optimized version of the library
-   make phip      -> Builds parallel hip optimized version of the library
-   make hipdebug  -> Builds serial hip debug version of the library
-   make phipdebug -> Builds parallel hip debug version of the library
+   make serial   -> Builds serial optimized version of the library
+   make parallel -> Builds parallel optimized version of the library
+   make debug    -> Builds serial debug version of the library
+   make pdebug   -> Builds parallel debug version of the library
 
 Note that any of the above shortcuts accept configuration options, either at the
 command line or through a user configuration file.
@@ -241,9 +193,8 @@ Configuration options (GNU make)
 See the configuration file config/defaults.mk for the default settings.
 
 Compilers:
-   CXX      - C++ compiler, serial build
-   MPICXX   - MPI C++ compiler, parallel build
-   CUDA_CXX - The CUDA compiler, 'nvcc'
+   CXX    - C++ compiler, serial build
+   MPICXX - MPI C++ compiler, parallel build
 
 Compiler options:
    OPTIM_FLAGS - Options for optimized build
@@ -279,7 +230,7 @@ MFEM_DEBUG = YES/NO
    and consistency checks that may simplify bug-hunting.
 
 MFEM_USE_EXCEPTIONS = YES/NO
-   Enable the use of exceptions. In particular, modifies the default behavior
+   Enable the use of exceptions. In particular, modifies the default bahavior
    when errors are encountered: throw an exception, instead of aborting.
 
 MFEM_USE_LIBUNWIND = YES/NO
@@ -299,12 +250,8 @@ MFEM_THREAD_SAFE = YES/NO
    Use thread-safe implementation for some classes/methods. This comes at the
    cost of extra memory allocation and de-allocation.
 
-MFEM_USE_LEGACY_OPENMP = YES/NO
-   Enable (basic) experimental OpenMP support. Requires MFEM_THREAD_SAFE.
-   This option is deprecated.
-
 MFEM_USE_OPENMP = YES/NO
-   Enable the OpenMP backend.
+   Enable (basic) experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 
 MFEM_USE_MEMALLOC = YES/NO
    Internal MFEM option: enable batch allocation for some small objects.
@@ -383,11 +330,11 @@ MFEM_USE_MPFR = YES/NO
    see below.
 
 MFEM_USE_SIDRE = YES/NO
-   Sidre is a component of LLNL's axom project, https://github.com/LLNL/axom,
-   that provides an HDF5-based file format for visualization or restart
-   capability following the Conduit (https://github.com/LLNL/conduit) mesh
-   blueprint specification. When enabled, this option requires installation of
-   HDF5 (see also MFEM_USE_NETCDF), Conduit and LLNL's axom project.
+   Sidre is a component of LLNL's axom project, http://goo.gl/cZyJdn, that
+   provides an HDF5-based file format for visualization or restart capability
+   following the Conduit (https://github.com/LLNL/conduit) mesh blueprint
+   specification. When enabled, this option requires installation of HDF5 (see
+   also MFEM_USE_NETCDF), Conduit and LLNL's axom project.
 
 MFEM_USE_CONDUIT = YES/NO
    Enables support for converting MFEM Mesh and Grid Function objects to and
@@ -415,33 +362,6 @@ MFEM_USE_PUMI = YES/NO
    models and effectively supports automated adaptive analysis. PUMI enables
    support for parallel unstructured mesh modifications in MFEM.
 
-MFEM_USE_CUDA = YES/NO
-   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
-   platform and programming model for general computing on graphical processing
-   units (GPUs). The variable CUDA_ARCH is used to specify the CUDA compute
-   capability used during compilation (by default, CUDA_ARCH=sm_60). When
-   enabled, this option uses the CUDA_* build options, see below.
-
-MFEM_USE_HIP = YES/NO
-   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
-   interface for portability developed by AMD that can target both AMD and
-   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
-   option uses the HIP_* build options, see below.
-
-MFEM_USE_RAJA = YES/NO
-   Enable support for the RAJA performance portability layer in MFEM. RAJA
-   provides a portable abstraction for loops, supporting different programming
-   model backends. When using RAJA built with CUDA support, CUDA support must be
-   also enabled in MFEM, i.e. MFEM_USE_CUDA=YES must be set.
-
-MFEM_USE_OCCA = YES/NO
-   Enables support for the OCCA library in MFEM. OCCA is an open-source library
-   which aims to make it easy to program different types of devices (e.g. CPU,
-   GPU, FPGA) by providing an unified API for interacting with JIT-compiled
-   backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
-   in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -463,7 +383,7 @@ directory and use the string @MFEM_DIR@, e.g. HYPRE_OPT = -I@MFEM_DIR@/../hypre.
 The specific libraries and their options are:
 
 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
-  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
+  URL: http://www.llnl.gov/CASC/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
@@ -477,8 +397,7 @@ The specific libraries and their options are:
        http://math-atlas.sourceforge.net (ATLAS)
   Options: LAPACK_OPT (currently not used/needed), LAPACK_LIB.
 
-- OpenMP (optional), usually part of compiler, used when either MFEM_USE_OPENMP
-  or MFEM_USE_LEGACY_OPENMP is set to YES.
+- OpenMP (optional), usually part of compiler, used when MFEM_USE_OPENMP = YES.
   Options: OPENMP_OPT, OPENMP_LIB.
 
 - High-resolution POSIX clocks: when using MFEM_TIMER_TYPE = 2, it may be
@@ -488,7 +407,6 @@ The specific libraries and their options are:
 - SUNDIALS (optional), used when MFEM_USE_SUNDIALS = YES.
   Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
   Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
-  Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
   If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
   both MPI and hypre.
   URL: http://computation.llnl.gov/projects/sundials/sundials-software
@@ -511,8 +429,7 @@ The specific libraries and their options are:
 
 - STRUMPACK (optional), used when MFEM_USE_STRUMPACK = YES. Note that STRUMPACK
   requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
-  includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
-  and PT-Scotch are optional dependencies.
+  includes METIS 5 in its distribution.
   The support for STRUMPACK was added in MFEM v3.3.2 and it requires STRUMPACK
   2.0.0 or later.
   URL: http://portal.nersc.gov/project/sparse/strumpack
@@ -543,8 +460,7 @@ The specific libraries and their options are:
   Options: PETSC_OPT, PETSC_LIB.
 
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
-  Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
-  URL: https://github.com/LLNL/axom
+  URL: http://goo.gl/cZyJdn (axom, to be released)
        https://github.com/LLNL/conduit (Conduit)
        https://support.hdfgroup.org/HDF5 (HDF5)
   Options: SIDRE_OPT, SIDRE_LIB.
@@ -559,23 +475,6 @@ The specific libraries and their options are:
   URL: https://scorec.rpi.edu/pumi
   Options: PUMI_OPT, PUMI_LIB.
 
-- CUDA, used when MFEM_USE_CUDA = YES.
-  URL: https://developer.nvidia.com/cuda-toolkit
-  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
-
-- HIP, used when MFEM_USE_HIP = YES.
-  URL: https://rocm.github.io/ROCmInstall.html
-  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
-
-- OCCA, used when MFEM_USE_OCCA = YES.
-  URL: https://libocca.org
-  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-
-- RAJA, used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
-  URL: https://github.com/LLNL/RAJA
-  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
   Options: MPFR_OPT, MPFR_LIB.
@@ -680,8 +579,6 @@ Configuration variables (CMake)
 ===============================
 See the configuration file config/defaults.cmake for the default settings.
 
-Note: the option MFEM_USE_CUDA requires CMake version 3.8 or newer!
-
 Non-standard CMake variables for compilers:
    CXX    - If set, overwrite the auto-detected C++ compiler, serial build
    MPICXX - If set, overwrite the auto-detected MPI C++ compiler, parallel build
@@ -699,7 +596,6 @@ MFEM_USE_METIS - Set to ${MFEM_USE_MPI}, can be overwritten.
 MFEM_USE_LIBUNWIND
 MFEM_USE_LAPACK
 MFEM_THREAD_SAFE
-MFEM_USE_LEGACY_OPENMP
 MFEM_USE_OPENMP
 MFEM_USE_MEMALLOC
 MFEM_TIMER_TYPE - Set automatically, can be overwritten.
@@ -712,10 +608,6 @@ MFEM_USE_NETCDF
 MFEM_USE_MPFR
 MFEM_USE_GZSTREAM
 MFEM_USE_PUMI
-MFEM_USE_CUDA
-MFEM_USE_OCCA
-MFEM_USE_RAJA
-MFEM_USE_SIDRE
 
 The following options are CMake specific:
 
@@ -762,9 +654,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - LIBUNWIND
  - POSIXCLOCKS
  - PUMI
- - OCCA
- - RAJA
- - AXOM - Used when MFEM_USE_SIDRE is enabled
 
 The following built-in CMake packages are also used:
 

README

@@ -8,9 +8,9 @@
                                http://mfem.org
 
 MFEM is a modular parallel C++ library for finite element methods. Its goal is
-to enable high-performance scalable finite element discretization research and
-application development on a wide variety of platforms, ranging from laptops to
-supercomputers.
+to enable the research and development of scalable finite element discretization
+and solver algorithms through general finite element abstractions, accurate and
+flexible visualization, and tight integration with the hypre library.
 
 * For building instructions, see the file INSTALL, or type "make help".
 
@@ -39,24 +39,23 @@ conforming and non-conforming (AMR) adaptive refinement. Arbitrary element
 transformations, allowing for high-order mesh elements with curved boundaries,
 are also supported.
 
-When used as a "finite element to linear algebra translator", MFEM can take a
-problem described in terms of finite element-type objects, and produce the
-corresponding linear algebra vectors and fully or partially assembled operators,
-e.g. in the form of global sparse matrices or matrix-free operators. The library
-includes simple smoothers and Krylov solvers, such as PCG, MINRES and GMRES, as
-well as support for sequential sparse direct solvers from the SuiteSparse
+MFEM is commonly used as a "finite element to linear algebra translator", since
+it can take a problem described in terms of finite element-type objects, and
+produce the corresponding linear algebra vectors and sparse matrices. In order
+to facilitate this, MFEM uses compressed sparse row (CSR) sparse matrix storage
+and includes simple smoothers and Krylov solvers, such as PCG, MINRES and GMRES,
+as well as support for sequential sparse direct solvers from the SuiteSparse
 library. Nonlinear solvers (the Newton method), eigensolvers (LOBPCG), and
 several explicit and implicit Runge-Kutta time integrators are also available.
 
 MFEM supports MPI-based parallelism throughout the library, and can readily be
-used as a scalable unstructured finite element problem generator. As of version
-4.0, MFEM offers initial support for GPU acceleration, and programming models,
-such as CUDA, OCCA, RAJA and OpenMP. MFEM-based applications require minimal
-changes to switch from a serial to a high-performing MPI-parallel version of the
-code, where they can take advantage of the integrated linear solvers from the
-hypre library.  Comprehensive support for other external packages, e.g. PETSc
-and SUNDIALS is also included, giving access to many additional linear and
-nonlinear solvers, preconditioners, time integrators, etc.
+used as a scalable unstructured finite element problem generator. MFEM-based
+applications require minimal changes to transition from a serial to a
+high-performing parallel version of the code, where they can take advantage of
+the integrated scalable linear solvers from the hypre library. Comprehensive
+support for other external packages, e.g. PETSc and SUNDIALS is also included,
+giving access to many additional linear and nonlinear solvers, preconditioners,
+time integrators, etc.
 
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
 as the OpenGL visualization tool GLVis which is available at http://glvis.org.

config/XSDKDefaults.cmake

@@ -74,7 +74,7 @@
 
 IF (NOT COMMAND PRINT_VAR)
   FUNCTION(PRINT_VAR  VAR_NAME)
-    MESSAGE(STATUS "${VAR_NAME} = '${${VAR_NAME}}'")
+    MESSAGE("-- " "${VAR_NAME} = '${${VAR_NAME}}'")
   ENDFUNCTION()
 ENDIF()
 
@@ -166,14 +166,14 @@ IF (USE_XSDK_DEFAULTS)
     ENDIF()
     XSDK_HANDLE_LANG_DEFAULTS(Fortran  FC  "FFLAGS;FCFLAGS")
   ENDIF()
-
+  
   # Set XSDK defaults for other CMake variables
-
+  
   IF ("${BUILD_SHARED_LIBS}"  STREQUAL  "")
     MESSAGE("-- " "XSDK: Setting default BUILD_SHARED_LIBS=TRUE")
     SET(BUILD_SHARED_LIBS  TRUE  CACHE  BOOL  "Set by default in XSDK mode")
   ENDIF()
-
+  
   IF ("${CMAKE_BUILD_TYPE}"  STREQUAL  "")
     MESSAGE("-- " "XSDK: Setting default CMAKE_BUILD_TYPE=DEBUG")
     SET(CMAKE_BUILD_TYPE  DEBUG  CACHE  STRING  "Set by default in XSDK mode")
@@ -181,13 +181,6 @@ IF (USE_XSDK_DEFAULTS)
 
 ENDIF()
 
-
-##################################################################################
-#
-#               MFEM-specific additions: set TPL MFEM_USE_* defaults
-#
-##################################################################################
-
 IF (DEFINED TPL_ENABLE_MPI)
   SET(MFEM_USE_MPI ${TPL_ENABLE_MPI} CACHE BOOL "Enable MPI parallel build" FORCE)
 ENDIF()
@@ -259,15 +252,3 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_PUMI)
   SET(MFEM_USE_PUMI ${TPL_ENABLE_PUMI} CACHE BOOL "Enable PUMI" FORCE)
 ENDIF()
-
-IF (DEFINED TPL_ENABLE_CUDA)
-  SET(MFEM_USE_CUDA ${TPL_ENABLE_CUDA} CACHE BOOL "Enable CUDA" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_OCCA)
-  SET(MFEM_USE_OCCA ${TPL_ENABLE_OCCA} CACHE BOOL "Enable OCCA" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_RAJA)
-  SET(MFEM_USE_RAJA ${TPL_ENABLE_RAJA} CACHE BOOL "Enable RAJA" FORCE)
-ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -25,7 +25,6 @@ set(MFEM_USE_LIBUNWIND @MFEM_USE_LIBUNWIND@)
 set(MFEM_USE_LAPACK @MFEM_USE_LAPACK@)
 set(MFEM_THREAD_SAFE @MFEM_THREAD_SAFE@)
 set(MFEM_USE_OPENMP @MFEM_USE_OPENMP@)
-set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
@@ -41,9 +40,6 @@ set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
-set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
-set(MFEM_USE_OCCA @MFEM_USE_OCCA@)
-set(MFEM_USE_RAJA @MFEM_USE_RAJA@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -30,12 +30,6 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)
 
-// MFEM source directory.
-#define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
-
-// MFEM install directory.
-#define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
-
 // Description of the git commit used to build MFEM.
 #cmakedefine MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
@@ -68,12 +62,9 @@
 // allocation and de-allocation.
 #cmakedefine MFEM_THREAD_SAFE
 
-// Enable the OpenMP backend.
+// Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 #cmakedefine MFEM_USE_OPENMP
 
-// [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
-#cmakedefine MFEM_USE_LEGACY_OPENMP
-
 // Enable MFEM functionality based on the Mesquite library.
 #cmakedefine MFEM_USE_MESQUITE
 
@@ -110,16 +101,6 @@
 // Enable MFEM functionality based on the PUMI library
 #cmakedefine MFEM_USE_PUMI
 
-// Build the GPU/CUDA-enabled version of the MFEM library.
-// Requires a CUDA compiler (nvcc).
-#cmakedefine MFEM_USE_CUDA
-
-// Enable MFEM functionality based on the RAJA library
-#cmakedefine MFEM_USE_RAJA
-
-// Enable MFEM functionality based on the OCCA library
-#cmakedefine MFEM_USE_OCCA
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.

config/cmake/modules/FindAxom.cmake

@@ -18,4 +18,6 @@ include(MfemCmakeUtilities)
 # Note: components are enabled based on the find_package() parameters.
 mfem_find_package(Axom AXOM AXOM_DIR "include" "" "lib" ""
   "Paths to headers required by Axom." "Libraries required by Axom."
-  ADD_COMPONENT Axom "include" axom/config.hpp "lib" axom)
+  ADD_COMPONENT Sidre "include" sidre/sidre.hpp "lib" sidre
+  ADD_COMPONENT SLIC "include" slic/slic.hpp "lib" slic
+  ADD_COMPONENT axom_utils "include" axom_utils/Utilities.hpp "lib" axom_utils)

config/cmake/modules/FindOCCA.cmake

@@ -1,19 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Defines the following variables:
-#   - OCCA_FOUND
-#   - OCCA_LIBRARIES
-#   - OCCA_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(OCCA OCCA OCCA_DIR "include" "occa.hpp" "lib" "occa"
-  "Paths to headers required by OCCA." "Libraries required by OCCA.")

config/cmake/modules/FindRAJA.cmake

@@ -1,30 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Defines the following variables:
-#   - RAJA_FOUND
-#   - RAJA_LIBRARIES
-#   - RAJA_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(RAJA RAJA RAJA_DIR "include" "RAJA/RAJA.hpp" "lib" "RAJA"
-  "Paths to headers required by RAJA." "Libraries required by RAJA.")
-
-if (NOT RAJA_CONFIG_CMAKE)
-   set(RAJA_CONFIG_CMAKE "${RAJA_DIR}/share/raja/cmake/raja-config.cmake")
-endif()
-if (EXISTS "${RAJA_CONFIG_CMAKE}")
-   include("${RAJA_CONFIG_CMAKE}")
-   if (ENABLE_CUDA AND NOT MFEM_USE_CUDA)
-      message(FATAL_ERROR
-         "RAJA is built with CUDA: MFEM_USE_CUDA=YES is required")
-   endif()
-endif()

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -232,12 +232,10 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
   # If we have the TPL_ versions of _INCLUDE_DIRS and _LIBRARIES then set the
   # standard ${Prefix} versions
   if (TPL_${Prefix}_INCLUDE_DIRS)
-    set(${Prefix}_INCLUDE_DIRS ${TPL_${Prefix}_INCLUDE_DIRS} CACHE STRING
-      "TPL_${Prefix}_INCLUDE_DIRS was found." FORCE)
+    set(${Prefix}_INCLUDE_DIRS ${TPL_${Prefix}_INCLUDE_DIRS} CACHE STRING "TPL_${Prefix}_INCLUDE_DIRS was found." FORCE)
   endif()
   if (TPL_${Prefix}_LIBRARIES)
-    set(${Prefix}_LIBRARIES ${TPL_${Prefix}_LIBRARIES} CACHE STRING
-      "TPL_${Prefix}_LIBRARIES was found." FORCE)
+    set(${Prefix}_LIBRARIES ${TPL_${Prefix}_LIBRARIES} CACHE STRING "TPL_${Prefix}_LIBRARIES was found." FORCE)
   endif()
 
   # Quick return
@@ -716,11 +714,11 @@ function(mfem_export_mk_files)
   # Convert Boolean vars to YES/NO without writting the values to cache
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_GZSTREAM MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GECKO MFEM_USE_GNUTLS
-      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA)
+      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_MEMALLOC
+      MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE MFEM_USE_SUPERLU
+      MFEM_USE_STRUMPACK MFEM_USE_GECKO MFEM_USE_GNUTLS MFEM_USE_NETCDF
+      MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT
+      MFEM_USE_PUMI)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -728,7 +726,6 @@ function(mfem_export_mk_files)
       set(${var} NO)
     endif()
   endforeach()
-  # TODO: Add support for MFEM_USE_CUDA=YES
   set(MFEM_CXX ${CMAKE_CXX_COMPILER})
   set(MFEM_CPPFLAGS "")
   string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"

config/config.hpp

@@ -10,15 +10,15 @@
 // Software Foundation) version 2.1 dated February 1999.
 
 
-// Support out-of-source builds: if MFEM_CONFIG_FILE is defined, include it.
+// Support out-of-source builds: if MFEM_BUILD_DIR is defined, load the config
+// file MFEM_BUILD_DIR/config/_config.hpp.
 //
 // Otherwise, use the local file: _config.hpp.
 
-#ifndef MFEM_CONFIG_HPP
-#define MFEM_CONFIG_HPP
-
-#ifdef MFEM_CONFIG_FILE
-#include MFEM_CONFIG_FILE
+#ifdef MFEM_BUILD_DIR
+#define MFEM_QUOTE(a) #a
+#define MFEM_MAKE_PATH(x,y) MFEM_QUOTE(x/y)
+#include MFEM_MAKE_PATH(MFEM_BUILD_DIR,config/_config.hpp)
 #else
 #include "_config.hpp"
 #endif
@@ -52,5 +52,3 @@
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
 #endif // MFEM_USE_MPI not defined
-
-#endif // MFEM_CONFIG_HPP

config/config.hpp.in

@@ -30,12 +30,6 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)
 
-// The absolute path of the MFEM source prefix
-// #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
-
-// The absolute path of the MFEM installation prefix
-// #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
-
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
@@ -68,12 +62,9 @@
 // allocation and de-allocation.
 // #define MFEM_THREAD_SAFE
 
-// Enable the OpenMP backend.
+// Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 // #define MFEM_USE_OPENMP
 
-// [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
-// #define MFEM_USE_LEGACY_OPENMP
-
 // Internal MFEM option: enable group/batch allocation for some small objects.
 // #define MFEM_USE_MEMALLOC
 
@@ -121,20 +112,6 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI
 
-// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
-// Requires a CUDA compiler (nvcc).
-// #define MFEM_USE_CUDA
-
-// Build the AMD GPU/HIP-enabled version of the MFEM library.
-// Requires a HIP compiler (hipcc).
-// #define MFEM_USE_HIP
-
-// Enable functionality based on the RAJA library.
-// #define MFEM_USE_RAJA
-
-// Enable functionality based on the OCCA library.
-// #define MFEM_USE_OCCA
-
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 

config/config.mk.in

@@ -10,41 +10,34 @@
 # Software Foundation) version 2.1 dated February 1999.
 
 # Variables corresponding to defines in config.hpp (YES, NO, or value)
-MFEM_VERSION           = @MFEM_VERSION@
-MFEM_VERSION_STRING    = @MFEM_VERSION_STRING@
-MFEM_SOURCE_DIR        = @MFEM_SOURCE_DIR@
-MFEM_INSTALL_DIR       = @MFEM_INSTALL_DIR@
-MFEM_GIT_STRING        = @MFEM_GIT_STRING@
-MFEM_USE_MPI           = @MFEM_USE_MPI@
-MFEM_USE_METIS         = @MFEM_USE_METIS@
-MFEM_USE_METIS_5       = @MFEM_USE_METIS_5@
-MFEM_DEBUG             = @MFEM_DEBUG@
-MFEM_USE_EXCEPTIONS    = @MFEM_USE_EXCEPTIONS@
-MFEM_USE_GZSTREAM      = @MFEM_USE_GZSTREAM@
-MFEM_USE_LIBUNWIND     = @MFEM_USE_LIBUNWIND@
-MFEM_USE_LAPACK        = @MFEM_USE_LAPACK@
-MFEM_THREAD_SAFE       = @MFEM_THREAD_SAFE@
-MFEM_USE_LEGACY_OPENMP = @MFEM_USE_LEGACY_OPENMP@
-MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
-MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
-MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
-MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
-MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
-MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
-MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
-MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
-MFEM_USE_GECKO         = @MFEM_USE_GECKO@
-MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
-MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
-MFEM_USE_PETSC         = @MFEM_USE_PETSC@
-MFEM_USE_MPFR          = @MFEM_USE_MPFR@
-MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
-MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
-MFEM_USE_PUMI          = @MFEM_USE_PUMI@
-MFEM_USE_CUDA          = @MFEM_USE_CUDA@
-MFEM_USE_HIP           = @MFEM_USE_HIP@
-MFEM_USE_RAJA          = @MFEM_USE_RAJA@
-MFEM_USE_OCCA          = @MFEM_USE_OCCA@
+MFEM_VERSION         = @MFEM_VERSION@
+MFEM_VERSION_STRING  = @MFEM_VERSION_STRING@
+MFEM_GIT_STRING      = @MFEM_GIT_STRING@
+MFEM_USE_MPI         = @MFEM_USE_MPI@
+MFEM_USE_METIS       = @MFEM_USE_METIS@
+MFEM_USE_METIS_5     = @MFEM_USE_METIS_5@
+MFEM_DEBUG           = @MFEM_DEBUG@
+MFEM_USE_EXCEPTIONS  = @MFEM_USE_EXCEPTIONS@
+MFEM_USE_GZSTREAM    = @MFEM_USE_GZSTREAM@
+MFEM_USE_LIBUNWIND   = @MFEM_USE_LIBUNWIND@
+MFEM_USE_LAPACK      = @MFEM_USE_LAPACK@
+MFEM_THREAD_SAFE     = @MFEM_THREAD_SAFE@
+MFEM_USE_OPENMP      = @MFEM_USE_OPENMP@
+MFEM_USE_MEMALLOC    = @MFEM_USE_MEMALLOC@
+MFEM_TIMER_TYPE      = @MFEM_TIMER_TYPE@
+MFEM_USE_SUNDIALS    = @MFEM_USE_SUNDIALS@
+MFEM_USE_MESQUITE    = @MFEM_USE_MESQUITE@
+MFEM_USE_SUITESPARSE = @MFEM_USE_SUITESPARSE@
+MFEM_USE_SUPERLU     = @MFEM_USE_SUPERLU@
+MFEM_USE_STRUMPACK   = @MFEM_USE_STRUMPACK@
+MFEM_USE_GECKO       = @MFEM_USE_GECKO@
+MFEM_USE_GNUTLS      = @MFEM_USE_GNUTLS@
+MFEM_USE_NETCDF      = @MFEM_USE_NETCDF@
+MFEM_USE_PETSC       = @MFEM_USE_PETSC@
+MFEM_USE_MPFR        = @MFEM_USE_MPFR@
+MFEM_USE_SIDRE       = @MFEM_USE_SIDRE@
+MFEM_USE_CONDUIT     = @MFEM_USE_CONDUIT@
+MFEM_USE_PUMI        = @MFEM_USE_PUMI@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -72,8 +65,5 @@ MFEM_MPIEXEC    = @MFEM_MPIEXEC@
 MFEM_MPIEXEC_NP = @MFEM_MPIEXEC_NP@
 MFEM_MPI_NP     = @MFEM_MPI_NP@
 
-# The NVCC compiler cannot link with -x=cu
-MFEM_LINK_FLAGS := $(filter-out -x=cu, $(MFEM_FLAGS))
-
 # Optional extra configuration
 @MFEM_CONFIG_EXTRA@

config/defaults.cmake

@@ -26,8 +26,7 @@ option(MFEM_USE_GZSTREAM "Enable gzstream for compressed data streams." OFF)
 option(MFEM_USE_LIBUNWIND "Enable backtrace for errors." OFF)
 option(MFEM_USE_LAPACK "Enable LAPACK usage" OFF)
 option(MFEM_THREAD_SAFE "Enable thread safety" OFF)
-option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
-option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
+option(MFEM_USE_OPENMP "Enable OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
@@ -42,9 +41,6 @@ option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
-option(MFEM_USE_CUDA "Enable CUDA" OFF)
-option(MFEM_USE_OCCA "Enable OCCA" OFF)
-option(MFEM_USE_RAJA "Enable RAJA" OFF)
 
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 
@@ -62,16 +58,13 @@ option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 # set(CXX g++)
 # set(MPICXX mpicxx)
 
-# Set the target CUDA architecture
-set(CUDA_ARCH "sm_60" CACHE STRING "Target CUDA architecture.")
-
 set(MFEM_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 # The *_DIR paths below will be the first place searched for the corresponding
 # headers and library. If these fail, then standard cmake search is performed.
 # Note: if the variables are already in the cache, they are not overwritten.
 
-set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
+set(HYPRE_DIR "${MFEM_DIR}/../hypre-2.10.0b/src/hypre" CACHE PATH
     "Path to the hypre library.")
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
@@ -81,7 +74,7 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."
 
 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")
 
-set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
+set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-3.0.0" CACHE PATH
     "Path to the SUNDIALS library.")
 # The following may be necessary, if SUNDIALS was built with KLU:
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
@@ -109,7 +102,6 @@ set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
     "Path to the STRUMPACK library.")
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
-# Starting with v2.2.0 of STRUMPACK, ParMETIS and Scotch are optional.
 set(STRUMPACK_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
     "ScaLAPACK" "Scotch/ptscotch/ptscotcherr/scotch/scotcherr" CACHE STRING
     "Additional packages required by STRUMPACK.")
@@ -117,8 +109,7 @@ set(STRUMPACK_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
 # set(STRUMPACK_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
 #     "Additional libraries required by STRUMPACK.")
 
-# The Scotch library, required by STRUMPACK <= v2.1.0, optional in STRUMPACK >=
-# v2.2.0.
+# The Scotch library, required by STRUMPACK
 set(Scotch_DIR "${MFEM_DIR}/../scotch_6.0.4" CACHE PATH
     "Path to the Scotch and PT-Scotch libraries.")
 set(Scotch_REQUIRED_PACKAGES "Threads" CACHE STRING
@@ -154,15 +145,12 @@ set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
 
 set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 # May need to add "Boost" as requirement.
-set(Axom_REQUIRED_PACKAGES "Conduit/relay/blueprint" CACHE STRING
+set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
     "Additional packages required by Axom.")
 
 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
     "Directory where PUMI is installed")
 
-set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
-set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")

config/defaults.mk

@@ -21,13 +21,8 @@ NOTMAC := $(subst Darwin,,$(shell uname -s))
 CXX = g++
 MPICXX = mpicxx
 
-BASE_FLAGS  = -std=c++11
-OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
-
-# Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
-CXX_XCOMPILER =
-CXX_XLINKER   = -Wl,
+OPTIM_FLAGS = -O3
+DEBUG_FLAGS = -g -Wall
 
 # Destination location of make install
 # PREFIX = $(HOME)/mfem
@@ -38,49 +33,33 @@ INSTALL = /usr/bin/install
 STATIC = YES
 SHARED = NO
 
-# CUDA configuration options
-CUDA_CXX = nvcc
-CUDA_ARCH = sm_60
-CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
-# Prefixes for passing flags to the host compiler and linker when using CUDA_CXX
-CUDA_XCOMPILER = -Xcompiler=
-CUDA_XLINKER   = -Xlinker=
-
-# HIP configuration options
-HIP_CXX = hipcc
-# The HIP_ARCH option specifies the AMD GPU processor, similar to CUDA_ARCH. For
-# example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo),
-# gfx900, gfx1010, etc.
-HIP_ARCH = gfx900
-HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
-
 ifneq ($(NOTMAC),)
    AR      = ar
    ARFLAGS = cruv
    RANLIB  = ranlib
-   PICFLAG = $(XCOMPILER)-fPIC
+   PICFLAG = -fPIC
    SO_EXT  = so
    SO_VER  = so.$(MFEM_VERSION_STRING)
-   BUILD_SOFLAGS = -shared $(XLINKER)-soname,libmfem.$(SO_VER)
-   BUILD_RPATH = $(XLINKER)-rpath,$(BUILD_REAL_DIR)
+   BUILD_SOFLAGS = -shared -Wl,-soname,libmfem.$(SO_VER)
+   BUILD_RPATH = -Wl,-rpath,$(BUILD_REAL_DIR)
    INSTALL_SOFLAGS = $(BUILD_SOFLAGS)
-   INSTALL_RPATH = $(XLINKER)-rpath,@MFEM_LIB_DIR@
+   INSTALL_RPATH = -Wl,-rpath,@MFEM_LIB_DIR@
 else
    # Silence "has no symbols" warnings on Mac OS X
    AR      = ar
    ARFLAGS = Scruv
    RANLIB  = ranlib -no_warning_for_no_symbols
-   PICFLAG = $(XCOMPILER)-fPIC
+   PICFLAG = -fPIC
    SO_EXT  = dylib
    SO_VER  = $(MFEM_VERSION_STRING).dylib
-   MAKE_SOFLAGS = $(XLINKER)-dylib,-install_name,$(1)/libmfem.$(SO_VER),\
+   MAKE_SOFLAGS = -Wl,-dylib,-install_name,$(1)/libmfem.$(SO_VER),\
       -compatibility_version,$(MFEM_VERSION_STRING),\
       -current_version,$(MFEM_VERSION_STRING),\
       -undefined,dynamic_lookup
    BUILD_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(BUILD_REAL_DIR)))
-   BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup
+   BUILD_RPATH = -Wl,-undefined,dynamic_lookup
    INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR)))
-   INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup
+   INSTALL_RPATH = -Wl,-undefined,dynamic_lookup
 endif
 
 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -103,36 +82,31 @@ MFEM_MPI_NP = 4
 # config.hpp. The values below are the defaults for generating the actual values
 # in config.mk and config.hpp.
 
-MFEM_USE_MPI           = NO
-MFEM_USE_METIS         = $(MFEM_USE_MPI)
-MFEM_USE_METIS_5       = NO
-MFEM_DEBUG             = NO
-MFEM_USE_EXCEPTIONS    = NO
-MFEM_USE_GZSTREAM      = NO
-MFEM_USE_LIBUNWIND     = NO
-MFEM_USE_LAPACK        = NO
-MFEM_THREAD_SAFE       = NO
-MFEM_USE_OPENMP        = NO
-MFEM_USE_LEGACY_OPENMP = NO
-MFEM_USE_MEMALLOC      = YES
-MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
-MFEM_USE_SUNDIALS      = NO
-MFEM_USE_MESQUITE      = NO
-MFEM_USE_SUITESPARSE   = NO
-MFEM_USE_SUPERLU       = NO
-MFEM_USE_STRUMPACK     = NO
-MFEM_USE_GECKO         = NO
-MFEM_USE_GNUTLS        = NO
-MFEM_USE_NETCDF        = NO
-MFEM_USE_PETSC         = NO
-MFEM_USE_MPFR          = NO
-MFEM_USE_SIDRE         = NO
-MFEM_USE_CONDUIT       = NO
-MFEM_USE_PUMI          = NO
-MFEM_USE_CUDA          = NO
-MFEM_USE_HIP           = NO
-MFEM_USE_RAJA          = NO
-MFEM_USE_OCCA          = NO
+MFEM_USE_MPI         = NO
+MFEM_USE_METIS       = $(MFEM_USE_MPI)
+MFEM_USE_METIS_5     = NO
+MFEM_DEBUG           = NO
+MFEM_USE_EXCEPTIONS  = NO
+MFEM_USE_GZSTREAM    = NO
+MFEM_USE_LIBUNWIND   = NO
+MFEM_USE_LAPACK      = NO
+MFEM_THREAD_SAFE     = NO
+MFEM_USE_OPENMP      = NO
+MFEM_USE_MEMALLOC    = YES
+MFEM_TIMER_TYPE      = $(if $(NOTMAC),2,4)
+MFEM_USE_SUNDIALS    = NO
+MFEM_USE_MESQUITE    = NO
+MFEM_USE_SUITESPARSE = NO
+MFEM_USE_SUPERLU     = NO
+MFEM_USE_STRUMPACK   = NO
+MFEM_USE_GECKO       = NO
+MFEM_USE_GNUTLS      = NO
+MFEM_USE_NETCDF      = NO
+MFEM_USE_PETSC       = NO
+MFEM_USE_MPFR        = NO
+MFEM_USE_SIDRE       = NO
+MFEM_USE_CONDUIT     = NO
+MFEM_USE_PUMI        = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -144,7 +118,7 @@ LIBUNWIND_OPT = -g
 LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 
 # HYPRE library configuration (needed to build the parallel version)
-HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
+HYPRE_DIR = @MFEM_DIR@/../hypre-2.10.0b/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
 
@@ -162,8 +136,6 @@ ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)
 else
    # ParMETIS: currently needed by SuperLU or STRUMPACK. We assume that METIS 5
    # (included with ParMETIS) is installed in the same location.
-   # Starting with STRUMPACK v2.2.0, ParMETIS is an optional dependency while
-   # METIS is still required.
    METIS_DIR = @MFEM_DIR@/../parmetis-4.0.3
    METIS_OPT = -I$(METIS_DIR)/include
    METIS_LIB = -L$(METIS_DIR)/lib -lparmetis -lmetis
@@ -175,16 +147,16 @@ LAPACK_OPT =
 LAPACK_LIB = $(if $(NOTMAC),-llapack -lblas,-framework Accelerate)
 
 # OpenMP configuration
-OPENMP_OPT = $(XCOMPILER)-fopenmp
+OPENMP_OPT = -fopenmp
 OPENMP_LIB =
 
 # Used when MFEM_TIMER_TYPE = 2
 POSIX_CLOCKS_LIB = -lrt
 
 # SUNDIALS library configuration
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-3.0.0
 SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
+SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib -L$(SUNDIALS_DIR)/lib\
  -lsundials_arkode -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol
 
 ifeq ($(MFEM_USE_MPI),YES)
@@ -209,10 +181,9 @@ SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
 # SuperLU library configuration
 SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
 SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
-SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
+SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/SRC -L$(SUPERLU_DIR)/SRC -lsuperlu_dist
 
-# SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
-# STRUMPACK >= v2.2.0)
+# SCOTCH library configuration (required by STRUMPACK)
 SCOTCH_DIR = @MFEM_DIR@/../scotch_6.0.4
 SCOTCH_OPT = -I$(SCOTCH_DIR)/include
 SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\
@@ -299,7 +270,7 @@ SIDRE_LIB = \
    -Wl,-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
    -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
    -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
-   -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl
+   -lsidre -lslic -laxom_utils -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl
 
 # PUMI
 # Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h
@@ -308,27 +279,6 @@ PUMI_OPT = -I$(PUMI_DIR)/include
 PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
    -llion -lmth -lapf_zoltan -lspr
 
-# CUDA library configuration (currently not needed)
-CUDA_OPT =
-CUDA_LIB =
-
-# HIP library configuration (currently not needed)
-HIP_OPT =
-HIP_LIB =
-
-# OCCA library configuration
-OCCA_DIR = @MFEM_DIR@/../occa
-OCCA_OPT = -I$(OCCA_DIR)/include
-OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
-
-# RAJA library configuration
-RAJA_DIR = @MFEM_DIR@/../raja
-RAJA_OPT = -I$(RAJA_DIR)/include
-ifdef CUB_DIR
-   RAJA_OPT += -I$(CUB_DIR)
-endif
-RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA
-
 # If YES, enable some informational messages
 VERBOSE = NO
 

config/makefile

@@ -36,7 +36,6 @@ CONFIG_MK = config.mk
 all: header config-mk
 
 MPI = $(MFEM_USE_MPI:NO=)
-GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
 GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
@@ -45,7 +44,7 @@ SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
-	$(GHV_CXX) ${GHV_FLAGS} $(SRC)$(GHV).cpp -o $(GHV)
+	$(MFEM_CXX) ${GHV_FLAGS} $(SRC)$(GHV).cpp -o $(GHV)
 $(GHV).out: $(GHV)
 	./$(GHV) > $(GHV).out
 .INTERMEDIATE: $(GHV) $(GHV).out

config/sample-runs.sh

@@ -18,8 +18,6 @@ run_prefix=""
 run_vg="valgrind --leak-check=full --show-reachable=yes --track-origins=yes"
 run_suffix="-no-vis"
 skip_gen_meshes="yes"
-# filter-out device runs ("no") or non-device runs ("yes"):
-device_runs="no"
 cur_dir="${PWD}"
 mfem_dir="$(cd "$(dirname "$0")"/.. && pwd)"
 mfem_build_dir=""
@@ -150,11 +148,6 @@ function extract_sample_runs()
    if [ "$skip_gen_meshes" == "yes" ]; then
       runs=`printf "%s" "$runs" | grep -v ".* -m .*\.gen"`
    fi
-   if [ "$device_runs" == "yes" ]; then
-      runs=`printf "%s" "$runs" | grep ".* -d .*"`
-   else
-      runs=`printf "%s" "$runs" | grep -v ".* -d .*"`
-   fi
    IFS=$'\n'
    runs=(${runs})
    IFS="${old_IFS}"
@@ -176,9 +169,6 @@ function help_message()
       -g <dir> <pattern>
                   Specify explicitly a group (dir + file pattern) to run; This
                   option can be used multiple times to define multiple groups
-      -dev        configure only sample runs using devices.
-                  To test with a parallel build, the parallel (-p|-par) option
-                  should be set first on the command line.
       -v          Enable valgrind
       -o <dir>    [${output_dir:-"<empty>: output goes to stdout"}]
                   If not empty, save output to files inside <dir>
@@ -263,7 +253,7 @@ case "$1" in
    -h|-help)
       opt_help="yes"
       ;;
-   -p|-par)
+   -p|-parallel)
       mfem_config="MFEM_USE_MPI=YES MFEM_DEBUG=NO"
       ;;
    -g)
@@ -274,10 +264,6 @@ case "$1" in
       groups=("${groups[@]}" "${test_group}")
       shift 2
       ;;
-   -dev)
-       device_runs="yes"
-       mfem_config+=" MFEM_USE_CUDA=YES MFEM_USE_OCCA=YES MFEM_USE_RAJA=YES MFEM_USE_OPENMP=YES"
-      ;;
    -v)
       valgrind="yes"
       ;;
@@ -308,10 +294,6 @@ case "$1" in
    -n)
       run_prefix="echo"
       ;;
-   -*)
-      echo "unknown option: '$1'"
-      exit 1
-      ;;
    *=*)
       eval $1
       ;;

config/tconfig.hpp

@@ -43,14 +43,15 @@
 #define MFEM_ALIGN_SIZE(size,type) \
    MFEM_ROUNDUP(size,(MFEM_SIMD_SIZE)/sizeof(type))
 
-#ifdef MFEM_COUNT_FLOPS
 namespace mfem
 {
 namespace internal
 {
-extern long long flop_count;
+long long flop_count;
 }
 }
+
+#ifdef MFEM_COUNT_FLOPS
 #define MFEM_FLOPS_RESET() (mfem::internal::flop_count = 0)
 #define MFEM_FLOPS_ADD(cnt) (mfem::internal::flop_count += (cnt))
 #define MFEM_FLOPS_GET() (mfem::internal::flop_count)

data/fichera.mesh

@@ -19,10 +19,10 @@ elements
 1 5 0 1 4 3 9 10 13 12
 1 5 3 4 7 6 12 13 16 15
 1 5 2 3 6 5 11 12 15 14
-1 5 11 12 15 14 20 21 24 23
-1 5 12 13 16 15 21 22 25 24
-1 5 9 10 13 12 18 19 22 21
 1 5 8 9 12 11 17 18 21 20
+1 5 9 10 13 12 18 19 22 21
+1 5 12 13 16 15 21 22 25 24
+1 5 11 12 15 14 20 21 24 23
 
 boundary
 24

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.0.1
+PROJECT_NUMBER         = v3.4.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/CodeDocumentation.dox

@@ -35,12 +35,6 @@ namespace mfem {
  * - HypreParMatrix and HypreParVector
  * - HypreSolver and other \link hypre.hpp hypre classes\endlink
  *
- * <H3>Main GPU classes</H3>
- * - Device
- * - Memory
- * - MemoryManager
- * - MFEM_FORALL macro in forall.hpp
- *
  * <H3>Example codes</H3>
  * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
  * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
@@ -79,8 +73,8 @@ namespace mfem {
  * - <a class="el" href="ex19p_8cpp_source.html">Example 19p</a>: parallel incompressible nonlinear elasticity
  * - <a class="el" href="ex20_8cpp_source.html">Example 20</a>: symplectic ODE integration
  * - <a class="el" href="ex20p_8cpp_source.html">Example 20p</a>: parallel symplectic ODE integration
- * - <a class="el" href="ex21_8cpp_source.html">Example 21</a>: adaptive mesh refinement for linear elasticity
- * - <a class="el" href="ex21p_8cpp_source.html">Example 21p</a>: parallel adaptive mesh refinement for linear elasticity
+ * - <a class="el" href="ex22_8cpp_source.html">Example 22</a>: adaptive mesh refinement for linear elasticity
+ * - <a class="el" href="ex22p_8cpp_source.html">Example 22p</a>: parallel adaptive mesh refinement for linear elasticity
  *
  * <H4>SUNDIALS Examples</H4>
  * - Variants of Examples
@@ -130,7 +124,6 @@ namespace mfem {
  * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
  * - <a class="el" href="load-dc_8cpp_source.html">Load DC</a>: visualize fields saved via DataCollection classes
  * - <a class="el" href="convert-dc_8cpp_source.html">Convert DC</a>: convert between diffirent DataCollection formats
- * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *

examples/CMakeLists.txt

@@ -27,7 +27,7 @@ list(APPEND ALL_EXE_SRCS
   ex18.cpp
   ex19.cpp
   ex20.cpp
-  ex21.cpp
+  ex22.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -52,7 +52,7 @@ if (MFEM_USE_MPI)
     ex18p.cpp
     ex19p.cpp
     ex20p.cpp
-    ex21p.cpp
+    ex22p.cpp
     )
 endif()
 
@@ -86,15 +86,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   endif()
 endforeach()
 
-# If STRUMPACK is enabled, add a test run that uses it.
-if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=4
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
-    ${MPIEXEC_POSTFLAGS})
-endif()
-
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
   add_subdirectory(sundials)

examples/ex1.cpp

@@ -25,14 +25,6 @@
 //               ex1 -m ../data/mobius-strip.mesh
 //               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
 //
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
 //               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
@@ -61,9 +53,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
+   bool visualization = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -73,10 +63,6 @@ int main(int argc, char *argv[])
                   " isoparametric space.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -88,18 +74,13 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
@@ -112,7 +93,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
    //    Lagrange finite elements of the specified order. If order < 1, we
    //    instead use an isoparametric/isogeometric space.
    FiniteElementCollection *fec;
@@ -133,7 +114,7 @@ int main(int argc, char *argv[])
    cout << "Number of finite element unknowns: "
         << fespace->GetTrueVSize() << endl;
 
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined by marking all
    //    the boundary attributes from the mesh as essential (Dirichlet) and
    //    converting them to a list of true dofs.
@@ -145,7 +126,7 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
    //    the basis functions in the finite element fespace.
    LinearForm *b = new LinearForm(fespace);
@@ -153,56 +134,48 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(one));
    b->Assemble();
 
-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero,
    //    which satisfies the boundary conditions.
    GridFunction x(fespace);
    x = 0.0;
 
-   // 9. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    domain integrator.
    BilinearForm *a = new BilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
 
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
+   // 9. Assemble the bilinear form and the corresponding linear system,
+   //    applying any necessary transformations such as: eliminating boundary
+   //    conditions, applying conforming constraints for non-conforming AMR,
+   //    static condensation, etc.
    if (static_cond) { a->EnableStaticCondensation(); }
    a->Assemble();
 
-   OperatorPtr A;
+   SparseMatrix A;
    Vector B, X;
    a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
 
-   cout << "Size of linear system: " << A->Height() << endl;
+   cout << "Size of linear system: " << A.Height() << endl;
 
-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
 #ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system A X = B with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
 #else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
+   // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
 #endif
-   }
-   else // No preconditioning for now in partial assembly mode.
-   {
-      CG(*A, B, X, 1, 2000, 1e-12, 0.0);
-   }
 
-   // 12. Recover the solution as a finite element grid function.
+   // 11. Recover the solution as a finite element grid function.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 13. Save the refined mesh and the solution. This output can be viewed later
+   // 12. Save the refined mesh and the solution. This output can be viewed later
    //     using GLVis: "glvis -m refined.mesh -g sol.gf".
    ofstream mesh_ofs("refined.mesh");
    mesh_ofs.precision(8);
@@ -211,7 +184,7 @@ int main(int argc, char *argv[])
    sol_ofs.precision(8);
    x.Save(sol_ofs);
 
-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -221,7 +194,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *mesh << x << flush;
    }
 
-   // 15. Free the used memory.
+   // 14. Free the used memory.
    delete a;
    delete b;
    delete fespace;

examples/ex11p.cpp

@@ -261,7 +261,8 @@ int main(int argc, char *argv[])
          strumpack->SetPrintSolveStatistics(false);
          strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
          strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         strumpack->DisableMatching();
+         strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+         // strumpack->SetSymmetricPattern(true);
          strumpack->SetOperator(*Arow);
          strumpack->SetFromCommandLine();
          precond = strumpack;

examples/ex12p.cpp

@@ -5,11 +5,11 @@
 // Sample runs:
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh
-//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
-//    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
-//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
+//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 79 -n 10 -o 2 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3876
+//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 79
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3876 -o 2 -sys
-//    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4526 -n 6 -o 3 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-hex-nurbs.mesh
 //

examples/ex17.cpp

@@ -252,7 +252,7 @@ int main(int argc, char *argv[])
    }
    else
    {
-      GMRES(A, M, B, X, 3, 5000, 100, rtol*rtol, 0.0);
+      GMRES(A, M, B, X, 3, 5000, 50, rtol*rtol, 0.0);
    }
 #else
    // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.

examples/ex19.cpp

@@ -144,7 +144,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options
-   const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/beam-hex.mesh";
    int ref_levels = 0;
    int order = 2;
    bool visualization = true;

examples/ex19p.cpp

@@ -150,7 +150,7 @@ int main(int argc, char *argv[])
    MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 
    // 2. Parse command-line options
-   const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/beam-hex.mesh";
    int ser_ref_levels = 0;
    int par_ref_levels = 0;
    int order = 2;

examples/ex1p.cpp

@@ -25,11 +25,6 @@
 //               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
 //               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
 //
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
 //               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
@@ -64,9 +59,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
+   bool visualization = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -76,10 +69,6 @@ int main(int argc, char *argv[])
                   " isoparametric space.");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -98,18 +87,13 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 10,000 elements.
@@ -122,7 +106,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -135,7 +119,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange finite elements of the specified order. If
    //    order < 1, we instead use an isoparametric/isogeometric space.
    FiniteElementCollection *fec;
@@ -162,7 +146,7 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined
    //    by marking all the boundary attributes from the mesh as essential
    //    (Dirichlet) and converting them to a list of true dofs.
@@ -174,7 +158,7 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system, which in this case is
    //    (1,phi_i) where phi_i are the basis functions in fespace.
    ParLinearForm *b = new ParLinearForm(fespace);
@@ -182,49 +166,49 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(one));
    b->Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero,
+   //    which satisfies the boundary conditions.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
    //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //     domain integrator.
    ParBilinearForm *a = new ParBilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
 
-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, static condensation, etc.
    if (static_cond) { a->EnableStaticCondensation(); }
    a->Assemble();
 
-   OperatorPtr A;
+   HypreParMatrix A;
    Vector B, X;
    a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
 
-   // 13. Solve the linear system A X = B.
-   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use no preconditioner, for now.
-   Solver *prec = NULL;
-   if (!pa) { prec = new HypreBoomerAMG; }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
 
-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   //     preconditioner from hypre.
+   HypreSolver *amg = new HypreBoomerAMG(A);
+   HyprePCG *pcg = new HyprePCG(A);
+   pcg->SetTol(1e-12);
+   pcg->SetMaxIter(200);
+   pcg->SetPrintLevel(2);
+   pcg->SetPreconditioner(*amg);
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 15. Save the refined mesh and the solution in parallel. This output can
+   // 14. Save the refined mesh and the solution in parallel. This output can
    //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_name;
@@ -240,7 +224,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -251,7 +235,9 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 17. Free the used memory.
+   // 16. Free the used memory.
+   delete pcg;
+   delete amg;
    delete a;
    delete b;
    delete fespace;

examples/ex22.cpp

@@ -1,16 +1,16 @@
-//                                MFEM Example 21
+//                                MFEM Example 22
 //
-// Compile with: make ex21
+// Compile with: make ex22
 //
-// Sample runs:  ex21
-//               ex21 -o 3
-//               ex21 -m ../data/beam-quad.mesh
-//               ex21 -m ../data/beam-quad.mesh -o 3
-//               ex21 -m ../data/beam-quad.mesh -o 3 -f 1
-//               ex21 -m ../data/beam-tet.mesh
-//               ex21 -m ../data/beam-tet.mesh -o 2
-//               ex21 -m ../data/beam-hex.mesh
-//               ex21 -m ../data/beam-hex.mesh -o 2
+// Sample runs:  ex22
+//               ex22 -o 3
+//               ex22 -m ../data/beam-quad.mesh
+//               ex22 -m ../data/beam-quad.mesh -o 3
+//               ex22 -m ../data/beam-quad.mesh -o 3 -f 1
+//               ex22 -m ../data/beam-tet.mesh
+//               ex22 -m ../data/beam-tet.mesh -o 2
+//               ex22 -m ../data/beam-hex.mesh
+//               ex22 -m ../data/beam-hex.mesh -o 2
 //
 // Description:  This is a version of Example 2 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the linear
@@ -287,11 +287,11 @@ int main(int argc, char *argv[])
    }
 
    {
-      ofstream mesh_ref_out("ex21_reference.mesh");
+      ofstream mesh_ref_out("ex22_reference.mesh");
       mesh_ref_out.precision(16);
       mesh.Print(mesh_ref_out);
 
-      ofstream mesh_out("ex21_deformed.mesh");
+      ofstream mesh_out("ex22_deformed.mesh");
       mesh_out.precision(16);
       GridFunction nodes(&fespace), *nodes_p = &nodes;
       mesh.GetNodes(nodes);
@@ -301,7 +301,7 @@ int main(int argc, char *argv[])
       mesh.Print(mesh_out);
       mesh.SwapNodes(nodes_p, own_nodes);
 
-      ofstream x_out("ex21_displacement.sol");
+      ofstream x_out("ex22_displacement.sol");
       x_out.precision(16);
       x.Save(x_out);
    }

examples/ex22p.cpp

@@ -1,15 +1,15 @@
-//                                MFEM Example 21
+//                                MFEM Example 22
 //
-// Compile with: make ex21p
+// Compile with: make ex22p
 //
-// Sample runs:  mpirun -np 4 ex21p
-//               mpirun -np 4 ex21p -o 3
-//               mpirun -np 4 ex21p -m ../data/beam-quad.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-quad.mesh -o 3
-//               mpirun -np 4 ex21p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-tet.mesh -o 2
-//               mpirun -np 4 ex21p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-hex.mesh -o 2
+// Sample runs:  mpirun -np 4 ex22p
+//               mpirun -np 4 ex22p -o 3
+//               mpirun -np 4 ex22p -m ../data/beam-quad.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-quad.mesh -o 3
+//               mpirun -np 4 ex22p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-tet.mesh -o 2
+//               mpirun -np 4 ex22p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-hex.mesh -o 2
 //
 // Description:  This is a version of Example 2p with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the linear
@@ -330,7 +330,7 @@ int main(int argc, char *argv[])
          x.Update();
       }
 
-      // 21. Inform also the bilinear and linear forms that the space has
+      // 22. Inform also the bilinear and linear forms that the space has
       //     changed.
       a.Update();
       b.Update();
@@ -338,9 +338,9 @@ int main(int argc, char *argv[])
 
    {
       ostringstream mref_name, mesh_name, sol_name;
-      mref_name << "ex21p_reference_mesh." << setfill('0') << setw(6) << myid;
-      mesh_name << "ex21p_deformed_mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "ex21p_displacement." << setfill('0') << setw(6) << myid;
+      mref_name << "ex22p_reference_mesh." << setfill('0') << setw(6) << myid;
+      mesh_name << "ex22p_deformed_mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "ex22p_displacement." << setfill('0') << setw(6) << myid;
 
       ofstream mesh_ref_out(mref_name.str().c_str());
       mesh_ref_out.precision(16);

examples/ex3p.cpp

@@ -102,7 +102,8 @@ int main(int argc, char *argv[])
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
    {
-      int ref_levels = (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels =
+         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
          mesh->UniformRefinement();

examples/ex6.cpp

@@ -15,11 +15,6 @@
 //               ex6 -m ../data/square-disc-surf.mesh -o 2
 //               ex6 -m ../data/amr-quad.mesh
 //
-// Device sample runs:
-//               ex6 -pa -d cuda
-//               ex6 -pa -d occa-cuda
-//               ex6 -pa -d raja-omp
-//
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
 //               equation -Delta u = 1 with homogeneous Dirichlet boundary
@@ -48,19 +43,13 @@ int main(int argc, char *argv[])
    // 1. Parse command-line options.
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
+   bool visualization = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -72,19 +61,14 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
    Mesh mesh(mesh_file, 1, 1);
    int dim = mesh.Dimension();
    int sdim = mesh.SpaceDimension();
 
-   // 4. Since a NURBS mesh can currently only be refined uniformly, we need to
+   // 3. Since a NURBS mesh can currently only be refined uniformly, we need to
    //    convert it to a piecewise-polynomial curved mesh. First we refine the
    //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
    if (mesh.NURBSext)
@@ -96,16 +80,15 @@ int main(int argc, char *argv[])
       mesh.SetCurvature(2);
    }
 
-   // 5. Define a finite element space on the mesh. The polynomial order is
+   // 4. Define a finite element space on the mesh. The polynomial order is
    //    one (linear) by default, but this can be changed on the command line.
    H1_FECollection fec(order, dim);
    FiniteElementSpace fespace(&mesh, &fec);
 
-   // 6. As in Example 1, we set up bilinear and linear forms corresponding to
+   // 5. As in Example 1, we set up bilinear and linear forms corresponding to
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
    BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    LinearForm b(&fespace);
 
    ConstantCoefficient one(1.0);
@@ -115,18 +98,18 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
 
-   // 7. The solution vector x and the associated finite element grid function
+   // 6. The solution vector x and the associated finite element grid function
    //    will be maintained over the AMR iterations. We initialize it to zero.
    GridFunction x(&fespace);
    x = 0.0;
 
-   // 8. All boundary attributes will be used for essential (Dirichlet) BC.
+   // 7. All boundary attributes will be used for essential (Dirichlet) BC.
    MFEM_VERIFY(mesh.bdr_attributes.Size() > 0,
                "Boundary attributes required in the mesh.");
    Array<int> ess_bdr(mesh.bdr_attributes.Max());
    ess_bdr = 1;
 
-   // 9. Connect to GLVis.
+   // 8. Connect to GLVis.
    char vishost[] = "localhost";
    int  visport   = 19916;
    socketstream sol_sock;
@@ -135,23 +118,23 @@ int main(int argc, char *argv[])
       sol_sock.open(vishost, visport);
    }
 
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     that uses the ComputeElementFlux method of the DiffusionIntegrator to
-   //     recover a smoothed flux (gradient) that is subtracted from the element
-   //     flux to get an error indicator. We need to supply the space for the
-   //     smoothed flux: an (H1)^sdim (i.e., vector-valued) space is used here.
+   // 9. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   //    that uses the ComputeElementFlux method of the DiffusionIntegrator to
+   //    recover a smoothed flux (gradient) that is subtracted from the element
+   //    flux to get an error indicator. We need to supply the space for the
+   //    smoothed flux: an (H1)^sdim (i.e., vector-valued) space is used here.
    FiniteElementSpace flux_fespace(&mesh, &fec, sdim);
    ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
    estimator.SetAnisotropic();
 
-   // 11. A refiner selects and refines elements based on a refinement strategy.
+   // 10. A refiner selects and refines elements based on a refinement strategy.
    //     The strategy here is to refine elements with errors larger than a
    //     fraction of the maximum element error. Other strategies are possible.
    //     The refiner will call the given error estimator.
    ThresholdRefiner refiner(estimator);
    refiner.SetTotalErrorFraction(0.7);
 
-   // 12. The main AMR loop. In each iteration we solve the problem on the
+   // 11. The main AMR loop. In each iteration we solve the problem on the
    //     current mesh, visualize the solution, and refine the mesh.
    const int max_dofs = 50000;
    for (int it = 0; ; it++)
@@ -160,53 +143,44 @@ int main(int argc, char *argv[])
       cout << "\nAMR iteration " << it << endl;
       cout << "Number of unknowns: " << cdofs << endl;
 
-      // 13. Assemble the right-hand side.
+      // 12. Assemble the stiffness matrix and the right-hand side.
+      a.Assemble();
       b.Assemble();
 
-      // 14. Set Dirichlet boundary values in the GridFunction x.
+      // 13. Set Dirichlet boundary values in the GridFunction x.
       //     Determine the list of Dirichlet true DOFs in the linear system.
       Array<int> ess_tdof_list;
       x.ProjectBdrCoefficient(zero, ess_bdr);
       fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-      // 15. Assemble the stiffness matrix.
-      a.Assemble();
-
-      // 16. Create the linear system: eliminate boundary conditions, constrain
+      // 14. Create the linear system: eliminate boundary conditions, constrain
       //     hanging nodes and possibly apply other transformations. The system
       //     will be solved for true (unconstrained) DOFs only.
-      OperatorPtr A;
+      SparseMatrix A;
       Vector B, X;
-
       const int copy_interior = 1;
       a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
 
-      // 17. Solve the linear system A X = B.
-      if (!pa)
-      {
 #ifndef MFEM_USE_SUITESPARSE
-         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-         GSSmoother M((SparseMatrix&)(*A));
-         PCG(*A, M, B, X, 3, 200, 1e-12, 0.0);
+      // 15. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+      //     solve the linear system with PCG.
+      GSSmoother M(A);
+      PCG(A, M, B, X, 3, 200, 1e-12, 0.0);
 #else
-         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-         UMFPackSolver umf_solver;
-         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-         umf_solver.SetOperator(*A);
-         umf_solver.Mult(B, X);
+      // 15. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
+      //     the linear system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(B, X);
 #endif
-      }
-      else // No preconditioning for now in partial assembly mode.
-      {
-         CG(*A, B, X, 3, 2000, 1e-12, 0.0);
-      }
 
-      // 18. After solving the linear system, reconstruct the solution as a
+      // 16. After solving the linear system, reconstruct the solution as a
       //     finite element GridFunction. Constrained nodes are interpolated
       //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
       a.RecoverFEMSolution(X, b, x);
 
-      // 19. Send solution by socket to the GLVis server.
+      // 17. Send solution by socket to the GLVis server.
       if (visualization && sol_sock.good())
       {
          sol_sock.precision(8);
@@ -219,7 +193,7 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 20. Call the refiner to modify the mesh. The refiner calls the error
+      // 18. Call the refiner to modify the mesh. The refiner calls the error
       //     estimator to obtain element errors, then it selects elements to be
       //     refined and finally it modifies the mesh. The Stop() method can be
       //     used to determine if a stopping criterion was met.
@@ -230,7 +204,7 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 21. Update the space to reflect the new state of the mesh. Also,
+      // 19. Update the space to reflect the new state of the mesh. Also,
       //     interpolate the solution x so that it lies in the new space but
       //     represents the same function. This saves solver iterations later
       //     since we'll have a good initial guess of x in the next step.
@@ -239,7 +213,7 @@ int main(int argc, char *argv[])
       fespace.Update();
       x.Update();
 
-      // 22. Inform also the bilinear and linear forms that the space has
+      // 20. Inform also the bilinear and linear forms that the space has
       //     changed.
       a.Update();
       b.Update();

examples/ex6p.cpp

@@ -15,11 +15,6 @@
 //               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex6p -pa -d cuda
-//               mpirun -np 4 ex6p -pa -d occa-cuda
-//               mpirun -np 4 ex6p -pa -d raja-omp
-//
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
 //               equation -Delta u = 1 with homogeneous Dirichlet boundary
@@ -54,8 +49,6 @@ int main(int argc, char *argv[])
    // 2. Parse command-line options.
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
-   bool pa = false;
-   const char *device_config = "cpu";
    bool visualization = true;
 
    OptionsParser args(argc, argv);
@@ -63,10 +56,6 @@ int main(int argc, char *argv[])
                   "Mesh file to use.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -85,19 +74,14 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
    int sdim = mesh->SpaceDimension();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
    //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
    //    sure that the mesh is non-conforming.
    if (mesh->NURBSext)
@@ -107,7 +91,7 @@ int main(int argc, char *argv[])
    }
    mesh->EnsureNCMesh();
 
-   // 6. Define a parallel mesh by partitioning the serial mesh.
+   // 5. Define a parallel mesh by partitioning the serial mesh.
    //    Once the parallel mesh is defined, the serial mesh can be deleted.
    ParMesh pmesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
@@ -117,16 +101,15 @@ int main(int argc, char *argv[])
    Array<int> ess_bdr(pmesh.bdr_attributes.Max());
    ess_bdr = 1;
 
-   // 7. Define a finite element space on the mesh. The polynomial order is
+   // 6. Define a finite element space on the mesh. The polynomial order is
    //    one (linear) by default, but this can be changed on the command line.
    H1_FECollection fec(order, dim);
    ParFiniteElementSpace fespace(&pmesh, &fec);
 
-   // 8. As in Example 1p, we set up bilinear and linear forms corresponding to
+   // 7. As in Example 1p, we set up bilinear and linear forms corresponding to
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
    ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    ParLinearForm b(&fespace);
 
    ConstantCoefficient one(1.0);
@@ -135,12 +118,12 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
 
-   // 9. The solution vector x and the associated finite element grid function
+   // 8. The solution vector x and the associated finite element grid function
    //    will be maintained over the AMR iterations. We initialize it to zero.
    ParGridFunction x(&fespace);
    x = 0;
 
-   // 10. Connect to GLVis.
+   // 9. Connect to GLVis.
    char vishost[] = "localhost";
    int  visport   = 19916;
 
@@ -162,7 +145,7 @@ int main(int argc, char *argv[])
       sout.precision(8);
    }
 
-   // 11. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
    //     with L2 projection in the smoothing step to better handle hanging
    //     nodes and parallel partitioning. We need to supply a space for the
    //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
@@ -176,14 +159,14 @@ int main(int argc, char *argv[])
    // ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec, dim);
    L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
 
-   // 12. A refiner selects and refines elements based on a refinement strategy.
+   // 11. A refiner selects and refines elements based on a refinement strategy.
    //     The strategy here is to refine elements with errors larger than a
    //     fraction of the maximum element error. Other strategies are possible.
    //     The refiner will call the given error estimator.
    ThresholdRefiner refiner(estimator);
    refiner.SetTotalErrorFraction(0.7);
 
-   // 13. The main AMR loop. In each iteration we solve the problem on the
+   // 12. The main AMR loop. In each iteration we solve the problem on the
    //     current mesh, visualize the solution, and refine the mesh.
    const int max_dofs = 100000;
    for (int it = 0; ; it++)
@@ -195,46 +178,41 @@ int main(int argc, char *argv[])
          cout << "Number of unknowns: " << global_dofs << endl;
       }
 
-      // 14. Assemble the right-hand side and determine the list of true
-      //     (i.e. parallel conforming) essential boundary dofs.
-      Array<int> ess_tdof_list;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      // 13. Assemble the stiffness matrix and the right-hand side. Note that
+      //     MFEM doesn't care at this point that the mesh is nonconforming
+      //     and parallel. The FE space is considered 'cut' along hanging
+      //     edges/faces, and also across processor boundaries.
+      a.Assemble();
       b.Assemble();
 
-      // 15. Assemble the stiffness matrix. Note that MFEM doesn't care at this
-      //     point that the mesh is nonconforming and parallel.  The FE space is
-      //     considered 'cut' along hanging edges/faces, and also across
-      //     processor boundaries.
-      a.Assemble();
-
-      // 16. Create the parallel linear system: eliminate boundary conditions.
+      // 14. Create the parallel linear system: eliminate boundary conditions,
+      //     constrain hanging nodes and nodes across processor boundaries.
       //     The system will be solved for true (unconstrained/unique) DOFs only.
-      OperatorPtr A;
-      Vector B, X;
+      Array<int> ess_tdof_list;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
+      HypreParMatrix A;
+      Vector B, X;
       const int copy_interior = 1;
       a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
 
-      // 17. Solve the linear system A X = B.
-      //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-      //     * With partial assembly, use no preconditioner, for now.
-      HypreBoomerAMG *amg = NULL;
-      if (!pa) { amg = new HypreBoomerAMG; amg->SetPrintLevel(0); }
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-6);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(3); // print the first and the last iterations only
-      if (amg) { cg.SetPreconditioner(*amg); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete amg;
+      // 15. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreBoomerAMG amg;
+      amg.SetPrintLevel(0);
+      CGSolver pcg(A.GetComm());
+      pcg.SetPreconditioner(amg);
+      pcg.SetOperator(A);
+      pcg.SetRelTol(1e-6);
+      pcg.SetMaxIter(200);
+      pcg.SetPrintLevel(3); // print the first and the last iterations only
+      pcg.Mult(B, X);
 
-      // 18. Switch back to the host and extract the parallel grid function
-      //     corresponding to the finite element approximation X. This is the
-      //     local solution on each processor.
+      // 16. Extract the parallel grid function corresponding to the finite element
+      //     approximation X. This is the local solution on each processor.
       a.RecoverFEMSolution(X, b, x);
 
-      // 19. Send the solution by socket to a GLVis server.
+      // 17. Send the solution by socket to a GLVis server.
       if (visualization)
       {
          sout << "parallel " << num_procs << " " << myid << "\n";
@@ -250,7 +228,7 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 20. Call the refiner to modify the mesh. The refiner calls the error
+      // 18. Call the refiner to modify the mesh. The refiner calls the error
       //     estimator to obtain element errors, then it selects elements to be
       //     refined and finally it modifies the mesh. The Stop() method can be
       //     used to determine if a stopping criterion was met.
@@ -264,7 +242,7 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 21. Update the finite element space (recalculate the number of DOFs,
+      // 19. Update the finite element space (recalculate the number of DOFs,
       //     etc.) and create a grid function update matrix. Apply the matrix
       //     to any GridFunctions over the space. In this case, the update
       //     matrix is an interpolation matrix so the updated GridFunction will
@@ -272,7 +250,7 @@ int main(int argc, char *argv[])
       fespace.Update();
       x.Update();
 
-      // 22. Load balance the mesh, and update the space and solution. Currently
+      // 20. Load balance the mesh, and update the space and solution. Currently
       //     available only for nonconforming meshes.
       if (pmesh.Nonconforming())
       {
@@ -284,7 +262,7 @@ int main(int argc, char *argv[])
          x.Update();
       }
 
-      // 23. Inform also the bilinear and linear forms that the space has
+      // 21. Inform also the bilinear and linear forms that the space has
       //     changed.
       a.Update();
       b.Update();

examples/ex9.cpp

@@ -132,8 +132,8 @@ int main(int argc, char *argv[])
 
    // 2. Read the mesh from the given mesh file. We can handle geometrically
    //    periodic meshes in this code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
 
    // 3. Define the ODE solver used for time integration. Several explicit
    //    Runge-Kutta methods are available.
@@ -147,6 +147,7 @@ int main(int argc, char *argv[])
       case 6: ode_solver = new RK6Solver; break;
       default:
          cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
          return 3;
    }
 
@@ -156,18 +157,18 @@ int main(int argc, char *argv[])
    //    a (piecewise-polynomial) high-order mesh.
    for (int lev = 0; lev < ref_levels; lev++)
    {
-      mesh.UniformRefinement();
+      mesh->UniformRefinement();
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
+   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
 
    // 5. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
    DG_FECollection fec(order, dim);
-   FiniteElementSpace fes(&mesh, &fec);
+   FiniteElementSpace fes(mesh, &fec);
 
    cout << "Number of unknowns: " << fes.GetVSize() << endl;
 
@@ -207,7 +208,7 @@ int main(int argc, char *argv[])
    {
       ofstream omesh("ex9.mesh");
       omesh.precision(precision);
-      mesh.Print(omesh);
+      mesh->Print(omesh);
       ofstream osol("ex9-init.gf");
       osol.precision(precision);
       u.Save(osol);
@@ -221,14 +222,14 @@ int main(int argc, char *argv[])
       if (binary)
       {
 #ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example9", &mesh);
+         dc = new SidreDataCollection("Example9", mesh);
 #else
          MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
 #endif
       }
       else
       {
-         dc = new VisItDataCollection("Example9", &mesh);
+         dc = new VisItDataCollection("Example9", mesh);
          dc->SetPrecision(precision);
       }
       dc->RegisterField("solution", &u);
@@ -253,7 +254,7 @@ int main(int argc, char *argv[])
       else
       {
          sout.precision(precision);
-         sout << "solution\n" << mesh << u;
+         sout << "solution\n" << *mesh << u;
          sout << "pause\n";
          sout << flush;
          cout << "GLVis visualization paused."
@@ -285,7 +286,7 @@ int main(int argc, char *argv[])
 
          if (visualization)
          {
-            sout << "solution\n" << mesh << u << flush;
+            sout << "solution\n" << *mesh << u << flush;
          }
 
          if (visit)

examples/makefile

@@ -22,9 +22,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
-  ex18 ex19 ex20 ex21
+  ex18 ex19 ex20 ex22
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
- ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p
+ ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex22p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -96,12 +96,6 @@ ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-e 1)
-# Testing: optional tests
-ifeq ($(MFEM_USE_STRUMPACK),YES)
-ex11p-test-strumpack: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
-test-par-YES: ex11p-test-strumpack
-endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 
@@ -125,4 +119,4 @@ clean-exec:
 	@rm -f vortex-mesh.* vortex.mesh vortex-?-init.* vortex-?-final.*
 	@rm -f deformation.* pressure.*
 	@rm -f ex20.dat ex20p_?????.dat gnuplot_ex20.inp gnuplot_ex20p.inp
-	@rm -f ex21*.mesh ex21*.sol ex21p_*.*
+	@rm -f ex22*.mesh ex22*.sol ex22p_*.*

examples/petsc/ex10p.cpp

@@ -27,11 +27,8 @@
 //               method HyperelasticOperator::ImplicitSolve is the only
 //               requirement for high-order implicit (SDIRK) time integration.
 //               If using PETSc to solve the nonlinear problem, use the option
-//               files provided (see rc_ex10p, rc_ex10p_mf, rc_ex10p_mfop) that
-//               customize the Newton-Krylov method.
-//               When option --jfnk is used, PETSc will use a Jacobian-free
-//               Newton-Krylov method, using a user-defined preconditioner
-//               constructed with the PetscPreconditionerFactory class.
+//               file provided (rc_ex10p) that customizes the
+//               Newton-Krylov method.
 //
 //               We recommend viewing examples 2 and 9 before viewing this
 //               example.
@@ -89,15 +86,12 @@ protected:
    Solver *J_solver;
    /// Preconditioner for the Jacobian solve in the Newton method
    Solver *J_prec;
-   /// Preconditioner factory for JFNK
-   PetscPreconditionerFactory *J_factory;
 
    mutable Vector z; // auxiliary vector
 
 public:
    HyperelasticOperator(ParFiniteElementSpace &f, Array<int> &ess_bdr,
-                        double visc, double mu, double K,
-                        bool use_petsc, bool petsc_use_jfnk);
+                        double visc, double mu, double K, bool use_petsc);
 
    /// Compute the right-hand side of the ODE system.
    virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
@@ -142,21 +136,8 @@ public:
    virtual Operator &GetGradient(const Vector &k) const;
 
    virtual ~ReducedSystemOperator();
-
 };
 
-/** Auxiliary class to provide preconditioners for matrix-free methods */
-class PreconditionerFactory : public PetscPreconditionerFactory
-{
-private:
-   // const ReducedSystemOperator& op; // unused for now (generates warning)
-
-public:
-   PreconditionerFactory(const ReducedSystemOperator& op_, const string& name_)
-      : PetscPreconditionerFactory(name_) /* , op(op_) */ {}
-   virtual mfem::Solver* NewPreconditioner(const mfem::OperatorHandle&);
-   virtual ~PreconditionerFactory() {}
-};
 
 /** Function representing the elastic energy density for the given hyperelastic
     model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */
@@ -206,7 +187,6 @@ int main(int argc, char *argv[])
    int vis_steps = 1;
    bool use_petsc = true;
    const char *petscrc_file = "";
-   bool petsc_use_jfnk = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -241,9 +221,6 @@ int main(int argc, char *argv[])
                   "Use or not PETSc to solve the nonlinear system.");
    args.AddOption(&petscrc_file, "-petscopts", "--petscopts",
                   "PetscOptions file to use.");
-   args.AddOption(&petsc_use_jfnk, "-jfnk", "--jfnk", "-no-jfnk",
-                  "--no-jfnk",
-                  "Use JFNK with user-defined preconditioner factory.");
    args.Parse();
    if (!args.Good())
    {
@@ -367,8 +344,7 @@ int main(int argc, char *argv[])
    // 9. Initialize the hyperelastic operator, the GLVis visualization and print
    //    the initial energies.
    HyperelasticOperator *oper = new HyperelasticOperator(fespace, ess_bdr, visc,
-                                                         mu, K, use_petsc,
-                                                         petsc_use_jfnk);
+                                                         mu, K, use_petsc);
 
    socketstream vis_v, vis_w;
    if (visualization)
@@ -544,7 +520,7 @@ Operator &ReducedSystemOperator::GetGradient(const Vector &k) const
    add(*v, dt, k, w);
    add(*x, dt, w, z);
    localJ->Add(dt*dt, H->GetLocalGradient(z));
-   // if we are using PETSc, the HypreParCSR Jacobian will be converted to
+   // if we are using PETSc, the HypreParCSR jacobian will be converted to
    // PETSc's AIJ on the fly
    Jacobian = M->ParallelAssemble(localJ);
    delete localJ;
@@ -561,8 +537,7 @@ ReducedSystemOperator::~ReducedSystemOperator()
 
 HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
                                            Array<int> &ess_bdr, double visc,
-                                           double mu, double K, bool use_petsc,
-                                           bool use_petsc_factory)
+                                           double mu, double K, bool use_petsc)
    : TimeDependentOperator(2*f.TrueVSize(), 0.0), fespace(f),
      M(&fespace), S(&fespace), H(&fespace),
      viscosity(visc), M_solver(f.GetComm()),
@@ -615,8 +590,6 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
       J_minres->SetPreconditioner(*J_prec);
       J_solver = J_minres;
 
-      J_factory = NULL;
-
       newton_solver.iterative_mode = false;
       newton_solver.SetSolver(*J_solver);
       newton_solver.SetOperator(*reduced_oper);
@@ -627,20 +600,12 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
    }
    else
    {
-      // if using PETSc, we create the same solver (Newton + MINRES + Jacobi)
+      // if using PETSc, we create the same solver (NEWTON+MINRES+Jacobi)
       // by command line options (see rc_ex10p)
       J_solver = NULL;
       J_prec = NULL;
-      J_factory = NULL;
       pnewton_solver = new PetscNonlinearSolver(f.GetComm(),
                                                 *reduced_oper);
-
-      // we can setup a factory to construct a "physics-based" preconditioner
-      if (use_petsc_factory)
-      {
-         J_factory = new PreconditionerFactory(*reduced_oper, "JFNK preconditioner");
-         pnewton_solver->SetPreconditionerFactory(J_factory);
-      }
       pnewton_solver->SetPrintLevel(1); // print Newton iterations
       pnewton_solver->SetRelTol(rel_tol);
       pnewton_solver->SetAbsTol(0.0);
@@ -726,26 +691,12 @@ HyperelasticOperator::~HyperelasticOperator()
 {
    delete J_solver;
    delete J_prec;
-   delete J_factory;
    delete reduced_oper;
    delete model;
    delete Mmat;
    delete pnewton_solver;
 }
 
-// This method gets called every time we need a preconditioner "oh"
-// contains the PetscParMatrix that wraps the operator constructed in
-// the GetGradient() method (see also PetscSolver::SetJacobianType()).
-// In this example, we just return a customizable PetscPreconditioner
-// using that matrix. However, the OperatorHandle argument can be
-// ignored, and any "physics-based" solver can be constructed since we
-// have access to the HyperElasticOperator class.
-Solver* PreconditionerFactory::NewPreconditioner(const mfem::OperatorHandle& oh)
-{
-   PetscParMatrix *pP;
-   oh.Get(pP);
-   return new PetscPreconditioner(*pP,"jfnk_");
-}
 
 double ElasticEnergyCoefficient::Eval(ElementTransformation &T,
                                       const IntegrationPoint &ip)
@@ -759,8 +710,8 @@ double ElasticEnergyCoefficient::Eval(ElementTransformation &T,
 
 void InitialDeformation(const Vector &x, Vector &y)
 {
-   // set the initial configuration to be the same as the reference,
-   // stress free, configuration
+   // set the initial configuration to be the same as the reference, stress
+   // free, configuration
    y = x;
 }
 

examples/petsc/makefile

@@ -84,10 +84,6 @@ EX9_E_ARGS       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts r
 EX9_ES_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
 EX9_IS_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
 EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
-EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
-EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
-EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
-
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
@@ -111,9 +107,6 @@ ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_IS_ARGS))
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/rc_ex10p_jfnk

@@ -1,5 +0,0 @@
-# matrix-free Jacobian action, preconditioner constructed using PetscPreconditionerFactory
--snes_monitor
--snes_mf_operator
--ksp_type minres
--jfnk_pc_type jacobi

examples/petsc/rc_ex10p_mf

@@ -1,4 +0,0 @@
-# matrix free -> no preconditioner
--snes_monitor
--snes_mf
--ksp_type minres

examples/petsc/rc_ex10p_mfop

@@ -1,5 +0,0 @@
-# matrix-free Jacobian action, preconditioner constructed from the matrix obtained by the GetGradient() method
--snes_monitor
--snes_mf_operator
--ksp_type minres
--pc_type jacobi

examples/sundials/CMakeLists.txt

@@ -42,12 +42,12 @@ add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)
 #   ctest -R sundials
 
 # Command line options for the tests.
-# Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
-set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
+# Example 9: test explicit CVODE time stepping
+set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 11)
 set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
 set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
-# Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
-set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
+# Example 10: test implicit CVODE time stepping
+set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -vs 10)
 set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
 set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
 # Example 16: use the default options

examples/sundials/ex10.cpp

@@ -4,16 +4,16 @@
 // Compile with: make ex10
 //
 // Sample runs:
-//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 12 -dt 0.15 -vs 10
-//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 16 -dt 0.3  -vs 5
-//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 12 -dt 0.2  -vs 5
+//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  5 -dt 0.15 -vs 10
+//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  7 -dt 0.3  -vs 5
+//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  5 -dt 0.2  -vs 5
 //    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  2 -dt 3 -nls kinsol
 //    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  2 -dt 3 -nls kinsol
 //    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  2 -dt 3 -nls kinsol
-//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 14 -dt 0.15 -vs 10
-//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 17 -dt 0.01 -vs 30
-//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 14 -dt 0.15 -vs 10
-//    ex10 -m ../../data/beam-quad-amr.mesh -r 2 -o 2 -s 12 -dt 0.15 -vs 10
+//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 15 -dt 5e-3 -vs 60
+//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 16 -dt 0.01 -vs 30
+//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 15 -dt 0.01 -vs 30
+//    ex10 -m ../../data/beam-quad-amr.mesh -r 2 -o 2 -s 5 -dt 0.15 -vs 10
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -53,6 +53,7 @@ using namespace std;
 using namespace mfem;
 
 class ReducedSystemOperator;
+class SundialsJacSolver;
 
 /** After spatial discretization, the hyperelastic model can be written as a
  *  system of ODEs:
@@ -91,17 +92,12 @@ protected:
 
    mutable Vector z; // auxiliary vector
 
-   SparseMatrix *grad_H;
-   SparseMatrix *Jacobian;
-
-   double saved_gamma; // saved gamma value from implicit setup
-
 public:
    /// Solver type to use in the ImplicitSolve() method, used by SDIRK methods.
    enum NonlinearSolverType
    {
       NEWTON = 0, ///< Use MFEM's plain NewtonSolver
-      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KINSolver)
+      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KinSolver)
    };
 
    HyperelasticOperator(FiniteElementSpace &f, Array<int> &ess_bdr,
@@ -110,41 +106,15 @@ public:
 
    /// Compute the right-hand side of the ODE system.
    virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
-
    /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
-
-   /// Custom Jacobian system solver for the SUNDIALS time integrators.
-   /** For the ODE system represented by HyperelasticOperator
-
-          M dv/dt = -(H(x) + S*v)
-          dx/dt = v,
-
-       this class facilitates the solution of linear systems of the form
-
-           (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
-                - γ yv +    yx =   bx
-
-       for given bv, bx, x, and γ = GetTimeStep(). */
-
-   /** Linear solve applicable to the SUNDIALS format.
-       Solves (Mass - dt J) y = Mass b, where in our case:
-       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
-              | 0  I |      |  I     0    |      | x_hat |      | b_x |
-       The result replaces the rhs b.
-       We substitute x_hat = b_x + dt v_hat and solve
-       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
-
-   /** Setup the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &y, const Vector &fy,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Connect the Jacobian linear system solver (SundialsJacSolver) used by
+       SUNDIALS' CVODE and ARKODE time integrators to the internal objects
+       created by HyperelasticOperator. This method is called by the InitSystem
+       method of SundialsJacSolver. */
+   void InitSundialsJacSolver(SundialsJacSolver &sjsolv);
 
    double ElasticEnergy(const Vector &x) const;
    double KineticEnergy(const Vector &v) const;
@@ -182,6 +152,53 @@ public:
    virtual ~ReducedSystemOperator();
 };
 
+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by HyperelasticOperator
+
+        M dv/dt = -(H(x) + S*v)
+          dx/dt = v,
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
+             - γ yv +    yx =   bx
+
+    for given bv, bx, x, and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   BilinearForm *M, *S;
+   NonlinearForm *H;
+   SparseMatrix *grad_H, *Jacobian;
+   Solver *J_solver;
+
+public:
+   SundialsJacSolver()
+      : M(), S(), H(), grad_H(), Jacobian(), J_solver() { }
+
+   /// Connect the solver to the objects created inside HyperelasticOperator.
+   void SetOperators(BilinearForm &M_, BilinearForm &S_,
+                     NonlinearForm &H_, Solver &solver)
+   {
+      M = &M_; S = &S_; H = &H_; J_solver = &solver;
+   }
+
+   /** Linear solve applicable to the SUNDIALS format.
+       Solves (Mass - dt J) y = Mass b, where in our case:
+       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
+              | 0  I |      |  I     0    |      | x_hat |      | b_x |
+       The result replaces the rhs b.
+       We substitute x_hat = b_x + dt v_hat and solve
+       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+
 
 /** Function representing the elastic energy density for the given hyperelastic
     model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */
@@ -226,12 +243,6 @@ int main(int argc, char *argv[])
 
    // Relative and absolute tolerances for CVODE and ARKODE.
    const double reltol = 1e-1, abstol = 1e-1;
-   // Since this example uses the loose tolerances defined above, it is
-   // necessary to lower the linear solver tolerance for CVODE which is relative
-   // to the above tolerances.
-   const double cvode_eps_lin = 1e-4;
-   // Similarly, the nonlinear tolerance for ARKODE needs to be tightened.
-   const double arkode_eps_nonlin = 1e-6;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -241,24 +252,15 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Backward Euler,\n\t"
-                  "2  - SDIRK2, L-stable\n\t"
-                  "3  - SDIRK3, L-stable\n\t"
-                  "4  - Implicit Midpoint,\n\t"
-                  "5  - SDIRK2, A-stable,\n\t"
-                  "6  - SDIRK3, A-stable,\n\t"
-                  "7  - Forward Euler,\n\t"
-                  "8  - RK2,\n\t"
-                  "9  - RK3 SSP,\n\t"
-                  "10 - RK4,\n\t"
-                  "11 - CVODE implicit BDF, approximate Jacobian,\n\t"
-                  "12 - CVODE implicit BDF, specified Jacobian,\n\t"
-                  "13 - CVODE implicit ADAMS, approximate Jacobian,\n\t"
-                  "14 - CVODE implicit ADAMS, specified Jacobian,\n\t"
-                  "15 - ARKODE implicit, approximate Jacobian,\n\t"
-                  "16 - ARKODE implicit, specified Jacobian,\n\t"
-                  "17 - ARKODE explicit, 4th order.");
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            4 - CVODE implicit, approximate Jacobian,\n\t"
+                  "            5 - CVODE implicit, specified Jacobian,\n\t"
+                  "            6 - ARKODE implicit, approximate Jacobian,\n\t"
+                  "            7 - ARKODE implicit, specified Jacobian,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4,\n\t"
+                  "            15 - CVODE (adaptive order) explicit,\n\t"
+                  "            16 - ARKODE default (4th order) explicit.");
    args.AddOption(&nls, "-nls", "--nonlinear-solver",
                   "Nonlinear systems solver: "
                   "\"newton\" (plain Newton) or \"kinsol\" (KINSOL).");
@@ -285,19 +287,72 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 17)
-   {
-      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      return 1;
-   }
-
    // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Setup the nonlinear solver
+   // 3. Define the ODE solver used for time integration. Several implicit
+   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //    explicit Runge-Kutta methods are available.
+   ODESolver *ode_solver;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver *sjsolver = NULL;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1: ode_solver = new BackwardEulerSolver; break;
+      case 2: ode_solver = new SDIRK23Solver(2); break;
+      case 3: ode_solver = new SDIRK33Solver; break;
+      case 4:
+      case 5:
+         cvode = new CVODESolver(CV_BDF, CV_NEWTON);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         if (ode_solver_type == 5)
+         {
+            sjsolver = new SundialsJacSolver;
+            cvode->SetLinearSolver(*sjsolver);
+         }
+         ode_solver = cvode; break;
+      case 6:
+      case 7:
+         arkode = new ARKODESolver(ARKODESolver::IMPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 7)
+         {
+            // Custom Jacobian inversion.
+            sjsolver = new SundialsJacSolver;
+            arkode->SetLinearSolver(*sjsolver);
+         }
+         ode_solver = arkode; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15:
+         cvode = new CVODESolver(CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 16:
+         arkode = new ARKODESolver(ARKODESolver::IMPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
    map<string,HyperelasticOperator::NonlinearSolverType> nls_map;
    nls_map["newton"] = HyperelasticOperator::NEWTON;
    nls_map["kinsol"] = HyperelasticOperator::KINSOL;
@@ -384,82 +439,11 @@ int main(int argc, char *argv[])
    cout << "initial kinetic energy (KE) = " << ke0 << endl;
    cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;
 
-   // 8. Define the ODE solver used for time integration. Several implicit
-   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //    explicit Runge-Kutta methods are available.
    double t = 0.0;
    oper.SetTime(t);
+   ode_solver->Init(oper);
 
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 4:  ode_solver = new ImplicitMidpointSolver; break;
-      case 5:  ode_solver = new SDIRK23Solver; break;
-      case 6:  ode_solver = new SDIRK34Solver; break;
-      // Explicit methods
-      case 7:  ode_solver = new ForwardEulerSolver; break;
-      case 8:  ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 9:  ode_solver = new RK3SSPSolver; break;
-      case 10: ode_solver = new RK4Solver; break;
-      // CVODE BDF
-      case 11:
-      case 12:
-         cvode = new CVODESolver(CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // CVODE Adams
-      case 13:
-      case 14:
-         cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 13)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // ARKStep Implicit methods
-      case 15:
-      case 16:
-         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 15)
-         {
-            arkode->UseSundialsLinearSolver();
-         }
-         ode_solver = arkode; break;
-      // ARKStep Explicit methods
-      case 17:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 11) { ode_solver->Init(oper); }
-
-   // 9. Perform time-integration (looping over the time iterations, ti, with a
+   // 8. Perform time-integration (looping over the time iterations, ti, with a
    //    time-step dt).
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
@@ -494,7 +478,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 10. Save the displaced mesh, the velocity and elastic energy.
+   // 9. Save the displaced mesh, the velocity and elastic energy.
    {
       v.SetFromTrueVector(); x.SetFromTrueVector();
       GridFunction *nodes = &x;
@@ -513,8 +497,9 @@ int main(int argc, char *argv[])
       w.Save(ee_ofs);
    }
 
-   // 11. Free the used memory.
+   // 10. Free the used memory.
    delete ode_solver;
+   delete sjsolver;
    delete mesh;
 
    return 0;
@@ -594,14 +579,81 @@ ReducedSystemOperator::~ReducedSystemOperator()
 }
 
 
+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+   HyperelasticOperator *he_oper;
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   he_oper = dynamic_cast<HyperelasticOperator*>(td_oper);
+   MFEM_VERIFY(he_oper, "operator is not HyperelasticOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // he_oper = static_cast<HyperelasticOperator*>(td_oper);
+
+   he_oper->InitSundialsJacSolver(*this);
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   int sc = y_pred.Size() / 2;
+   const Vector x(y_pred.GetData() + sc, sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // J = M + dt*(S + dt*grad(H))
+   delete Jacobian;
+   Jacobian = Add(1.0, M->SpMat(), dt, S->SpMat());
+   grad_H = dynamic_cast<SparseMatrix *>(&H->GetGradient(x));
+   Jacobian->Add(dt * dt, *grad_H);
+
+   J_solver->SetOperator(*Jacobian);
+
+   jac_cur = 1;
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   int sc = b.Size() / 2;
+   // Vector x(y_cur.GetData() + sc, sc);
+   Vector b_v(b.GetData() +  0, sc);
+   Vector b_x(b.GetData() + sc, sc);
+   Vector rhs(sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // rhs = M b_v - dt*grad(H) b_x
+   grad_H->Mult(b_x, rhs);
+   rhs *= -dt;
+   M->AddMult(b_v, rhs);
+
+   J_solver->iterative_mode = false;
+   J_solver->Mult(rhs, b_v);
+
+   b_x.Add(dt, b_v);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   delete Jacobian;
+   return 0;
+}
+
+
 HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
                                            Array<int> &ess_bdr, double visc,
                                            double mu, double K,
                                            NonlinearSolverType nls_type)
    : TimeDependentOperator(2*f.GetTrueVSize(), 0.0), fespace(f),
      M(&fespace), S(&fespace), H(&fespace),
-     viscosity(visc), z(height/2),
-     grad_H(NULL), Jacobian(NULL)
+     viscosity(visc), z(height/2)
 {
    const double rel_tol = 1e-8;
    const int skip_zero_entries = 0;
@@ -650,24 +702,23 @@ HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
 
    if (nls_type == KINSOL)
    {
-      KINSolver *kinsolver = new KINSolver(KIN_NONE, true);
+      KinSolver *kinsolver = new KinSolver(KIN_NONE, true);
+      kinsolver->SetMaxSetupCalls(4);
       newton_solver = kinsolver;
-      newton_solver->SetOperator(*reduced_oper);
       newton_solver->SetMaxIter(200);
       newton_solver->SetRelTol(rel_tol);
       newton_solver->SetPrintLevel(0);
-      kinsolver->SetMaxSetupCalls(4);
    }
    else
    {
       newton_solver = new NewtonSolver();
-      newton_solver->SetOperator(*reduced_oper);
       newton_solver->SetMaxIter(10);
       newton_solver->SetRelTol(rel_tol);
       newton_solver->SetPrintLevel(-1);
    }
    newton_solver->SetSolver(*J_solver);
    newton_solver->iterative_mode = false;
+   newton_solver->SetOperator(*reduced_oper);
 }
 
 void HyperelasticOperator::Mult(const Vector &vx, Vector &dvx_dt) const
@@ -717,53 +768,9 @@ void HyperelasticOperator::ImplicitSolve(const double dt,
    add(v, dt, dv_dt, dx_dt);
 }
 
-int HyperelasticOperator::SUNImplicitSetup(const Vector &y,
-                                           const Vector &fy, int jok, int *jcur,
-                                           double gamma)
+void HyperelasticOperator::InitSundialsJacSolver(SundialsJacSolver &sjsolv)
 {
-   int sc = y.Size() / 2;
-   const Vector x(y.GetData() + sc, sc);
-
-   // J = M + dt*(S + dt*grad(H))
-   if (Jacobian) { delete Jacobian; }
-   Jacobian = Add(1.0, M.SpMat(), gamma, S.SpMat());
-   grad_H = dynamic_cast<SparseMatrix *>(&H.GetGradient(x));
-   Jacobian->Add(gamma * gamma, *grad_H);
-
-   // Set Jacobian solve operator
-   J_solver->SetOperator(*Jacobian);
-
-   // Indicate that the Jacobian was updated
-   *jcur = 1;
-
-   // Save gamma for use in solve
-   saved_gamma = gamma;
-
-   // Return success
-   return 0;
-}
-
-int HyperelasticOperator::SUNImplicitSolve(const Vector &b, Vector &x,
-                                           double tol)
-{
-   int sc = b.Size() / 2;
-   Vector b_v(b.GetData() +  0, sc);
-   Vector b_x(b.GetData() + sc, sc);
-   Vector x_v(x.GetData() +  0, sc);
-   Vector x_x(x.GetData() + sc, sc);
-   Vector rhs(sc);
-
-   // rhs = M b_v - dt*grad(H) b_x
-   grad_H->Mult(b_x, rhs);
-   rhs *= -saved_gamma;
-   M.AddMult(b_v, rhs);
-
-   J_solver->iterative_mode = false;
-   J_solver->Mult(rhs, x_v);
-
-   add(b_x, saved_gamma, x_v, x_x);
-
-   return 0;
+   sjsolv.SetOperators(M, S, H, *J_solver);
 }
 
 double HyperelasticOperator::ElasticEnergy(const Vector &x) const
@@ -785,7 +792,6 @@ void HyperelasticOperator::GetElasticEnergyDensity(
 
 HyperelasticOperator::~HyperelasticOperator()
 {
-   delete Jacobian;
    delete newton_solver;
    delete J_solver;
    delete J_prec;

examples/sundials/ex10p.cpp

@@ -4,16 +4,16 @@
 // Compile with: make ex10p
 //
 // Sample runs:
-//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 12 -dt 0.15 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 16 -dt 0.25 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 12 -dt 0.15 -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s  5 -dt 0.15 -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s  7 -dt 0.25  -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s  5 -dt 0.15  -vs 10
 //    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s  2 -dt 3 -nls kinsol
 //    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s  2 -dt 3 -nls kinsol
 //    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rs 1 -o 2 -s  2 -dt 3 -nls kinsol
-//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 14 -dt 0.15 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 17 -dt 5e-3 -vs 60
-//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 14 -dt 0.15 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-quad-amr.mesh -rp 1 -o 2 -s 12 -dt 0.15 -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 15 -dt 3e-3 -vs 120
+//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 16 -dt 5e-3 -vs 60
+//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 15 -dt 5e-3 -vs 60
+//    mpirun -np 4 ex10p -m ../../data/beam-quad-amr.mesh -rp 1 -o 2 -s 5 -dt 0.15 -vs 10
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -53,6 +53,7 @@ using namespace std;
 using namespace mfem;
 
 class ReducedSystemOperator;
+class SundialsJacSolver;
 
 /** After spatial discretization, the hyperelastic model can be written as a
  *  system of ODEs:
@@ -93,17 +94,12 @@ protected:
 
    mutable Vector z; // auxiliary vector
 
-   const SparseMatrix *local_grad_H;
-   HypreParMatrix *Jacobian;
-
-   double saved_gamma; // saved gamma value from implicit setup
-
 public:
    /// Solver type to use in the ImplicitSolve() method, used by SDIRK methods.
    enum NonlinearSolverType
    {
       NEWTON = 0, ///< Use MFEM's plain NewtonSolver
-      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KINSolver)
+      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KinSolver)
    };
 
    HyperelasticOperator(ParFiniteElementSpace &f, Array<int> &ess_bdr,
@@ -112,41 +108,15 @@ public:
 
    /// Compute the right-hand side of the ODE system.
    virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
-
    /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
 
-
-   /// Custom Jacobian system solver for the SUNDIALS time integrators.
-   /** For the ODE system represented by HyperelasticOperator
-
-          M dv/dt = -(H(x) + S*v)
-          dx/dt = v,
-
-       this class facilitates the solution of linear systems of the form
-
-           (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
-                - γ yv +    yx =   bx
-
-       for given bv, bx, x, and γ = GetTimeStep(). */
-
-   /** Linear solve applicable to the SUNDIALS format.
-       Solves (Mass - dt J) y = Mass b, where in our case:
-       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
-              | 0  I |      |  I     0    |      | x_hat |      | b_x |
-       The result replaces the rhs b.
-       We substitute x_hat = b_x + dt v_hat and solve
-       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
-
-   /** Setup the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &y, const Vector &fy,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Connect the Jacobian linear system solver (SundialsJacSolver) used by
+       SUNDIALS' CVODE and ARKODE time integrators to the internal objects
+       created by HyperelasticOperator. This method is called by the InitSystem
+       method of SundialsJacSolver. */
+   void InitSundialsJacSolver(SundialsJacSolver &sjsolv);
 
    double ElasticEnergy(const ParGridFunction &x) const;
    double KineticEnergy(const ParGridFunction &v) const;
@@ -187,6 +157,57 @@ public:
    virtual ~ReducedSystemOperator();
 };
 
+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by HyperelasticOperator
+
+        M dv/dt = -(H(x) + S*v)
+          dx/dt = v,
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
+             - γ yv +    yx =   bx
+
+    for given bv, bx, x, and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   ParBilinearForm *M, *S;
+   ParNonlinearForm *H;
+   const SparseMatrix *local_grad_H;
+   HypreParMatrix *Jacobian;
+   Solver *J_solver;
+   const Array<int> *ess_tdof_list;
+
+public:
+   SundialsJacSolver()
+      : M(), S(), H(), local_grad_H(), Jacobian(), J_solver() { }
+
+   /// Connect the solver to the objects created inside HyperelasticOperator.
+   void SetOperators(ParBilinearForm &M_, ParBilinearForm &S_,
+                     ParNonlinearForm &H_, Solver &solver,
+                     const Array<int> &ess_tdof_list_)
+   {
+      M = &M_; S = &S_; H = &H_; J_solver = &solver;
+      ess_tdof_list = &ess_tdof_list_;
+   }
+
+   /** Linear solve applicable to the SUNDIALS format.
+       Solves (Mass - dt J) y = Mass b, where in our case:
+       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
+              | 0  I |      |  I     0    |      | x_hat |      | b_x |
+       The result replaces the rhs b.
+       We substitute x_hat = b_x + dt v_hat and solve
+       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+
 
 /** Function representing the elastic energy density for the given hyperelastic
     model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */
@@ -238,12 +259,6 @@ int main(int argc, char *argv[])
 
    // Relative and absolute tolerances for CVODE and ARKODE.
    const double reltol = 1e-1, abstol = 1e-1;
-   // Since this example uses the loose tolerances defined above, it is
-   // necessary to lower the linear solver tolerance for CVODE which is relative
-   // to the above tolerances.
-   const double cvode_eps_lin = 1e-4;
-   // Similarly, the nonlinear tolerance for ARKODE needs to be tightened.
-   const double arkode_eps_nonlin = 1e-6;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -255,24 +270,15 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Backward Euler,\n\t"
-                  "2  - SDIRK2, L-stable\n\t"
-                  "3  - SDIRK3, L-stable\n\t"
-                  "4  - Implicit Midpoint,\n\t"
-                  "5  - SDIRK2, A-stable,\n\t"
-                  "6  - SDIRK3, A-stable,\n\t"
-                  "7  - Forward Euler,\n\t"
-                  "8  - RK2,\n\t"
-                  "9  - RK3 SSP,\n\t"
-                  "10 - RK4,\n\t"
-                  "11 - CVODE implicit BDF, approximate Jacobian,\n\t"
-                  "12 - CVODE implicit BDF, specified Jacobian,\n\t"
-                  "13 - CVODE implicit ADAMS, approximate Jacobian,\n\t"
-                  "14 - CVODE implicit ADAMS, specified Jacobian,\n\t"
-                  "15 - ARKODE implicit, approximate Jacobian,\n\t"
-                  "16 - ARKODE implicit, specified Jacobian,\n\t"
-                  "17 - ARKODE explicit, 4th order.");
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            4 - CVODE implicit, approximate Jacobian,\n\t"
+                  "            5 - CVODE implicit, specified Jacobian,\n\t"
+                  "            6 - ARKODE implicit, approximate Jacobian,\n\t"
+                  "            7 - ARKODE implicit, specified Jacobian,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4,\n\t"
+                  "            15 - CVODE (adaptive order) explicit,\n\t"
+                  "            16 - ARKODE default (4th order) explicit.");
    args.AddOption(&nls, "-nls", "--nonlinear-solver",
                   "Nonlinear systems solver: "
                   "\"newton\" (plain Newton) or \"kinsol\" (KINSOL).");
@@ -306,24 +312,76 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 17)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      }
-      MPI_Finalize();
-      return 1;
-   }
-
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Nonlinear solver
+   // 4. Define the ODE solver used for time integration. Several implicit
+   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //    explicit Runge-Kutta methods are available.
+   ODESolver *ode_solver;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver *sjsolver = NULL;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1: ode_solver = new BackwardEulerSolver; break;
+      case 2: ode_solver = new SDIRK23Solver(2); break;
+      case 3: ode_solver = new SDIRK33Solver; break;
+      case 4:
+      case 5:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF, CV_NEWTON);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         if (ode_solver_type == 5)
+         {
+            sjsolver = new SundialsJacSolver;
+            cvode->SetLinearSolver(*sjsolver); // Custom Jacobian inversion.
+         }
+         ode_solver = cvode; break;
+      case 6:
+      case 7:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::IMPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 7)
+         {
+            sjsolver = new SundialsJacSolver;
+            arkode->SetLinearSolver(*sjsolver); // Custom Jacobian inversion.
+         }
+         ode_solver = arkode; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 16:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         if (myid == 0)
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         MPI_Finalize();
+         return 3;
+   }
+
    map<string,HyperelasticOperator::NonlinearSolverType> nls_map;
    nls_map["newton"] = HyperelasticOperator::NEWTON;
    nls_map["kinsol"] = HyperelasticOperator::KINSOL;
@@ -333,6 +391,7 @@ int main(int argc, char *argv[])
       {
          cout << "Unknown type of nonlinear solver: " << nls << endl;
       }
+      delete ode_solver;
       delete mesh;
       MPI_Finalize();
       return 4;
@@ -436,82 +495,11 @@ int main(int argc, char *argv[])
       cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;
    }
 
-   // 10. Define the ODE solver used for time integration. Several implicit
-   //     singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //     explicit Runge-Kutta methods are available.
    double t = 0.0;
    oper.SetTime(t);
+   ode_solver->Init(oper);
 
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 4:  ode_solver = new ImplicitMidpointSolver; break;
-      case 5:  ode_solver = new SDIRK23Solver; break;
-      case 6:  ode_solver = new SDIRK34Solver; break;
-      // Explicit methods
-      case 7:  ode_solver = new ForwardEulerSolver; break;
-      case 8:  ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 9:  ode_solver = new RK3SSPSolver; break;
-      case 10: ode_solver = new RK4Solver; break;
-      // CVODE BDF
-      case 11:
-      case 12:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // CVODE Adams
-      case 13:
-      case 14:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 13)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // ARKStep Implicit methods
-      case 15:
-      case 16:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 15)
-         {
-            arkode->UseSundialsLinearSolver();
-         }
-         ode_solver = arkode; break;
-      // ARKStep Explicit methods
-      case 17:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 11) { ode_solver->Init(oper); }
-
-   // 11. Perform time-integration
+   // 10. Perform time-integration
    //     (looping over the time iterations, ti, with a time-step dt).
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
@@ -550,7 +538,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 12. Save the displaced mesh, the velocity and elastic energy.
+   // 11. Save the displaced mesh, the velocity and elastic energy.
    {
       v_gf.SetFromTrueVector(); x_gf.SetFromTrueVector();
       GridFunction *nodes = &x_gf;
@@ -575,8 +563,9 @@ int main(int argc, char *argv[])
       w_gf.Save(ee_ofs);
    }
 
-   // 13. Free the used memory.
+   // 12. Free the used memory.
    delete ode_solver;
+   delete sjsolver;
    delete pmesh;
 
    MPI_Finalize();
@@ -664,14 +653,92 @@ ReducedSystemOperator::~ReducedSystemOperator()
 }
 
 
+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+   HyperelasticOperator *he_oper;
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   he_oper = dynamic_cast<HyperelasticOperator*>(td_oper);
+   MFEM_VERIFY(he_oper, "operator is not HyperelasticOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // he_oper = static_cast<HyperelasticOperator*>(td_oper);
+
+   he_oper->InitSundialsJacSolver(*this);
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   int sc = y_pred.Size() / 2;
+   const Vector x(y_pred.GetData() + sc, sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // J = M + dt*(S + dt*grad(H))
+   delete Jacobian;
+   SparseMatrix *localJ = Add(1.0, M->SpMat(), dt, S->SpMat());
+   local_grad_H = &H->GetLocalGradient(x);
+   localJ->Add(dt*dt, *local_grad_H);
+   Jacobian = M->ParallelAssemble(localJ);
+   delete localJ;
+   HypreParMatrix *Je = Jacobian->EliminateRowsCols(*ess_tdof_list);
+   delete Je;
+
+   J_solver->SetOperator(*Jacobian);
+
+   jac_cur = 1;
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   int sc = b.Size() / 2;
+   ParFiniteElementSpace *fes = H->ParFESpace();
+   // Vector x(y_cur.GetData() + sc, sc);
+   Vector b_v(b.GetData() +  0, sc);
+   Vector b_x(b.GetData() + sc, sc);
+   Vector rhs(sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // We can assume that b_v and b_x have zeros at essential tdofs.
+
+   // rhs = M b_v - dt*grad(H) b_x
+   ParGridFunction lb_x(fes), lrhs(fes);
+   lb_x.Distribute(b_x);
+   local_grad_H->Mult(lb_x, lrhs);
+   lrhs.ParallelAssemble(rhs);
+   rhs *= -dt;
+   M->TrueAddMult(b_v, rhs);
+   rhs.SetSubVector(*ess_tdof_list, 0.0);
+
+   J_solver->iterative_mode = false;
+   J_solver->Mult(rhs, b_v);
+
+   b_x.Add(dt, b_v);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   delete Jacobian;
+   return 0;
+}
+
+
 HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
                                            Array<int> &ess_bdr, double visc,
                                            double mu, double K,
                                            NonlinearSolverType nls_type)
    : TimeDependentOperator(2*f.TrueVSize(), 0.0), fespace(f),
      M(&fespace), S(&fespace), H(&fespace),
-     viscosity(visc), M_solver(f.GetComm()), z(height/2),
-     local_grad_H(NULL), Jacobian(NULL)
+     viscosity(visc), M_solver(f.GetComm()), z(height/2)
 {
    const double rel_tol = 1e-8;
    const int skip_zero_entries = 0;
@@ -721,24 +788,23 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
 
    if (nls_type == KINSOL)
    {
-      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_NONE, true);
+      KinSolver *kinsolver = new KinSolver(f.GetComm(), KIN_NONE, true);
+      kinsolver->SetMaxSetupCalls(4);
       newton_solver = kinsolver;
-      newton_solver->SetOperator(*reduced_oper);
       newton_solver->SetMaxIter(200);
       newton_solver->SetRelTol(rel_tol);
       newton_solver->SetPrintLevel(0);
-      kinsolver->SetMaxSetupCalls(4);
    }
    else
    {
       newton_solver = new NewtonSolver(f.GetComm());
-      newton_solver->SetOperator(*reduced_oper);
       newton_solver->SetMaxIter(10);
       newton_solver->SetRelTol(rel_tol);
       newton_solver->SetPrintLevel(-1);
    }
    newton_solver->SetSolver(*J_solver);
    newton_solver->iterative_mode = false;
+   newton_solver->SetOperator(*reduced_oper);
 }
 
 void HyperelasticOperator::Mult(const Vector &vx, Vector &dvx_dt) const
@@ -792,64 +858,9 @@ void HyperelasticOperator::ImplicitSolve(const double dt,
    add(v, dt, dv_dt, dx_dt);
 }
 
-int HyperelasticOperator::SUNImplicitSetup(const Vector &y,
-                                           const Vector &fy, int jok, int *jcur,
-                                           double gamma)
+void HyperelasticOperator::InitSundialsJacSolver(SundialsJacSolver &sjsolv)
 {
-   int sc = y.Size() / 2;
-   const Vector x(y.GetData() + sc, sc);
-
-   // J = M + dt*(S + dt*grad(H))
-   if (Jacobian) { delete Jacobian; }
-   SparseMatrix *localJ = Add(1.0, M.SpMat(), gamma, S.SpMat());
-   local_grad_H = &H.GetLocalGradient(x);
-   localJ->Add(gamma*gamma, *local_grad_H);
-   Jacobian = M.ParallelAssemble(localJ);
-   delete localJ;
-   HypreParMatrix *Je = Jacobian->EliminateRowsCols(ess_tdof_list);
-   delete Je;
-
-   // Set Jacobian solve operator
-   J_solver->SetOperator(*Jacobian);
-
-   // Indicate that the Jacobian was updated
-   *jcur = 1;
-
-   // Save gamma for use in solve
-   saved_gamma = gamma;
-
-   // Return success
-   return 0;
-}
-
-int HyperelasticOperator::SUNImplicitSolve(const Vector &b, Vector &x,
-                                           double tol)
-{
-   int sc = b.Size() / 2;
-   ParFiniteElementSpace *fes = H.ParFESpace();
-   Vector b_v(b.GetData() +  0, sc);
-   Vector b_x(b.GetData() + sc, sc);
-   Vector x_v(x.GetData() +  0, sc);
-   Vector x_x(x.GetData() + sc, sc);
-   Vector rhs(sc);
-
-   // We can assume that b_v and b_x have zeros at essential tdofs.
-
-   // rhs = M b_v - dt*grad(H) b_x
-   ParGridFunction lb_x(fes), lrhs(fes);
-   lb_x.Distribute(b_x);
-   local_grad_H->Mult(lb_x, lrhs);
-   lrhs.ParallelAssemble(rhs);
-   rhs *= -saved_gamma;
-   M.TrueAddMult(b_v, rhs);
-   rhs.SetSubVector(ess_tdof_list, 0.0);
-
-   J_solver->iterative_mode = false;
-   J_solver->Mult(rhs, x_v);
-
-   add(b_x, saved_gamma, x_v, x_x);
-
-   return 0;
+   sjsolv.SetOperators(M, S, H, *J_solver, ess_tdof_list);
 }
 
 double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const
@@ -875,7 +886,6 @@ void HyperelasticOperator::GetElasticEnergyDensity(
 
 HyperelasticOperator::~HyperelasticOperator()
 {
-   delete Jacobian;
    delete newton_solver;
    delete J_solver;
    delete J_prec;

examples/sundials/ex16.cpp

@@ -7,9 +7,9 @@
 //               ex16 -m ../../data/inline-tri.mesh
 //               ex16 -m ../../data/disc-nurbs.mesh -tf 2
 //               ex16 -s 12 -a 0.0 -k 1.0
-//               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
-//               ex16 -s 9 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
-//               ex16 -s 10 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               ex16 -s 1 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
+//               ex16 -s 2 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
+//               ex16 -s 3 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               ex16 -m ../../data/fichera-q2.mesh
 //               ex16 -m ../../data/escher.mesh
 //               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -58,6 +58,7 @@ protected:
 
    SparseMatrix Mmat, Kmat;
    SparseMatrix *T; // T = M + dt K
+   double current_dt;
 
    CGSolver M_solver; // Krylov solver for inverting the mass matrix M
    DSmoother M_prec;  // Preconditioner for the mass matrix M
@@ -74,30 +75,13 @@ public:
                       const Vector &u);
 
    virtual void Mult(const Vector &u, Vector &du_dt) const;
-
    /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
    virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
 
-   /// Custom Jacobian system solver for the SUNDIALS time integrators.
-   /** For the ODE system represented by ConductionOperator
-
-           M du/dt = -K(u),
-
-       this class facilitates the solution of linear systems of the form
-
-           (M + γK) y = M b,
-
-       for given b, u (not used), and γ = GetTimeStep(). */
-
-   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Solve the system (M + dt K) y = M b. The result y replaces the input b.
+       This method is used by the implicit SUNDIALS solvers. */
+   void SundialsSolve(const double dt, Vector &b);
 
    /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
    void SetParameters(const Vector &u);
@@ -105,6 +89,33 @@ public:
    virtual ~ConductionOperator();
 };
 
+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by ConductionOperator
+
+        M du/dt = -K(u),
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γK) y = M b,
+
+    for given b, u (not used), and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   ConductionOperator *oper;
+
+public:
+   SundialsJacSolver() : oper(NULL) { }
+
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+
 double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
@@ -113,7 +124,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../../data/star.mesh";
    int ref_levels = 2;
    int order = 2;
-   int ode_solver_type = 9; // CVODE implicit BDF
+   int ode_solver_type = 11; // 11 = CVODE implicit
    double t_final = 0.5;
    double dt = 1.0e-2;
    double alpha = 1.0e-2;
@@ -136,19 +147,12 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Forward Euler,\n\t"
-                  "2  - RK2,\n\t"
-                  "3  - RK3 SSP,\n\t"
-                  "4  - RK4,\n\t"
-                  "5  - Backward Euler,\n\t"
-                  "6  - SDIRK 2,\n\t"
-                  "7  - SDIRK 3,\n\t"
-                  "8  - CVODE (implicit Adams),\n\t"
-                  "9  - CVODE (implicit BDF),\n\t"
-                  "10 - ARKODE (default explicit),\n\t"
-                  "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default impicit).");
+                  "ODE solver:\n"
+                  "\t 1/11 - CVODE (explicit/implicit),\n"
+                  "\t 2/12 - ARKODE (default explicit/implicit),\n"
+                  "\t 3 - ARKODE (Fehlberg-6-4-5)\n"
+                  "\t 4 - Forward Euler, 5 - RK2, 6 - RK3 SSP, 7 - RK4,\n"
+                  "\t 8 - Backward Euler, 9 - SDIRK23, 10 - SDIRK33.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -171,11 +175,6 @@ int main(int argc, char *argv[])
       args.PrintUsage(cout);
       return 1;
    }
-   if (ode_solver_type < 1 || ode_solver_type > 12)
-   {
-      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      return 3;
-   }
    args.PrintOptions(cout);
 
    // 2. Read the mesh from the given mesh file. We can handle triangular,
@@ -183,7 +182,61 @@ int main(int argc, char *argv[])
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 3. Define the ODE solver used for time integration. Several
+   // SUNDIALS solvers are available, as well as included both
+   // explicit and implicit MFEM ODE solvers.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver sun_solver; // Used by the implicit SUNDIALS ode solvers.
+   switch (ode_solver_type)
+   {
+      // SUNDIALS solvers
+      case 1:
+         cvode = new CVODESolver(CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 11:
+         cvode = new CVODESolver(CV_BDF, CV_NEWTON);
+         cvode->SetLinearSolver(sun_solver);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 2:
+      case 3:
+         arkode = new ARKODESolver(ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 3) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKODESolver(ARKODESolver::IMPLICIT);
+         arkode->SetLinearSolver(sun_solver);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Other MFEM explicit methods
+      case 4: ode_solver = new ForwardEulerSolver; break;
+      case 5: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 6: ode_solver = new RK3SSPSolver; break;
+      case 7: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 8:  ode_solver = new BackwardEulerSolver; break;
+      case 9:  ode_solver = new SDIRK23Solver(2); break;
+      case 10: ode_solver = new SDIRK33Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // Since we want to update the diffusion coefficient after every time step,
+   // we need to use the "one-step" mode of the SUNDIALS solvers.
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
    //    command-line parameter.
    for (int lev = 0; lev < ref_levels; lev++)
@@ -191,7 +244,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 4. Define the vector finite element space representing the current and the
+   // 5. Define the vector finite element space representing the current and the
    //    initial temperature, u_ref.
    H1_FECollection fe_coll(order, dim);
    FiniteElementSpace fespace(mesh, &fe_coll);
@@ -201,14 +254,14 @@ int main(int argc, char *argv[])
 
    GridFunction u_gf(&fespace);
 
-   // 5. Set the initial conditions for u. All boundaries are considered
+   // 6. Set the initial conditions for u. All boundaries are considered
    //    natural.
    FunctionCoefficient u_0(InitialTemperature);
    u_gf.ProjectCoefficient(u_0);
    Vector u;
    u_gf.GetTrueDofs(u);
 
-   // 6. Initialize the conduction operator and the visualization.
+   // 7. Initialize the conduction operator and the visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
 
    u_gf.SetFromTrueDofs(u);
@@ -254,65 +307,13 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define the ODE solver used for time integration.
-   double t = 0.0;
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // MFEM explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      // MFEM implicit L-stable methods
-      case 5: ode_solver = new BackwardEulerSolver; break;
-      case 6: ode_solver = new SDIRK23Solver(2); break;
-      case 7: ode_solver = new SDIRK33Solver; break;
-      // CVODE
-      case 8:
-         cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      case 9:
-         cvode = new CVODESolver(CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      // ARKODE
-      case 10:
-      case 11:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-      case 12:
-         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 8) { ode_solver->Init(oper); }
-
-   // Since we want to update the diffusion coefficient after every time step,
-   // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
-   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
-
    // 8. Perform time-integration (looping over the time iterations, ti, with a
    //    time-step dt).
    cout << "Integrating the ODE ..." << endl;
    tic_toc.Clear();
    tic_toc.Start();
+   ode_solver->Init(oper);
+   double t = 0.0;
 
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
@@ -370,7 +371,7 @@ int main(int argc, char *argv[])
 ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
                                        double kap, const Vector &u)
    : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
-     T(NULL), z(height)
+     T(NULL), current_dt(0.0), z(height)
 {
    const double rel_tol = 1e-8;
 
@@ -416,14 +417,32 @@ void ConductionOperator::ImplicitSolve(const double dt,
    // Solve the equation:
    //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
    // for du_dt
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, dt, Kmat);
-   T_solver.SetOperator(*T);
+   if (!T)
+   {
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
    Kmat.Mult(u, z);
    z.Neg();
    T_solver.Mult(z, du_dt);
 }
 
+void ConductionOperator::SundialsSolve(const double dt, Vector &b)
+{
+   // Solve the system (M + dt K) y = M b. The result y replaces the input b.
+   if (!T || dt != current_dt)
+   {
+      delete T;
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, b);
+}
+
 void ConductionOperator::SetParameters(const Vector &u)
 {
    GridFunction u_alpha_gf(&fespace);
@@ -441,26 +460,8 @@ void ConductionOperator::SetParameters(const Vector &u)
    K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
    K->Assemble();
    K->FormSystemMatrix(ess_tdof_list, Kmat);
-}
-
-int ConductionOperator::SUNImplicitSetup(const Vector &x,
-                                         const Vector &fx, int jok, int *jcur,
-                                         double gamma)
-{
-   // Setup the ODE Jacobian T = M + gamma K.
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, gamma, Kmat);
-   T_solver.SetOperator(*T);
-   *jcur = 1;
-   return (0);
-}
-
-int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
-{
-   // Solve the system A x = z => (M - gamma K) x = M b.
-   Mmat.Mult(b, z);
-   T_solver.Mult(z, x);
-   return (0);
+   delete T;
+   T = NULL; // re-compute T on the next ImplicitSolve or SundialsSolve
 }
 
 ConductionOperator::~ConductionOperator()
@@ -470,6 +471,46 @@ ConductionOperator::~ConductionOperator()
    delete K;
 }
 
+
+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   oper = dynamic_cast<ConductionOperator*>(td_oper);
+   MFEM_VERIFY(oper, "operator is not ConductionOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // oper = static_cast<ConductionOperator*>(td_oper);
+
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   jac_cur = 1;
+
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   oper->SundialsSolve(GetTimeStep(sundials_mem), b);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   return 0;
+}
+
+
 double InitialTemperature(const Vector &x)
 {
    if (x.Norml2() < 0.5)

examples/sundials/ex16p.cpp

@@ -8,9 +8,9 @@
 //     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
 //     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
 //     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
-//     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
-//     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
-//     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
+//     mpirun -np 4 ex16p -s 1 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
+//     mpirun -np 8 ex16p -s 2 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
+//     mpirun -np 4 ex16p -s 3 -dt 2.0e-4 -tf 4.0e-2
 //     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
 //     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
 //     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -77,19 +77,13 @@ public:
                       const Vector &u);
 
    virtual void Mult(const Vector &u, Vector &du_dt) const;
-
    /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
        This is the only requirement for high-order SDIRK implicit integration.*/
    virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
 
-   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Solve the system (M + dt K) y = M b. The result y replaces the input b.
+       This method is used by the implicit SUNDIALS solvers. */
+   void SundialsSolve(const double dt, Vector &b);
 
    /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
    void SetParameters(const Vector &u);
@@ -97,6 +91,33 @@ public:
    virtual ~ConductionOperator();
 };
 
+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by ConductionOperator
+
+        M du/dt = -K(u),
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γK) y = M b,
+
+    for given b, u (not used), and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   ConductionOperator *oper;
+
+public:
+   SundialsJacSolver() : oper(NULL) { }
+
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+
 double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
@@ -112,7 +133,7 @@ int main(int argc, char *argv[])
    int ser_ref_levels = 2;
    int par_ref_levels = 1;
    int order = 2;
-   int ode_solver_type = 9; // CVODE implicit BDF
+   int ode_solver_type = 11; // 11 = CVODE implicit
    double t_final = 0.5;
    double dt = 1.0e-2;
    double alpha = 1.0e-2;
@@ -137,19 +158,12 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Forward Euler,\n\t"
-                  "2  - RK2,\n\t"
-                  "3  - RK3 SSP,\n\t"
-                  "4  - RK4,\n\t"
-                  "5  - Backward Euler,\n\t"
-                  "6  - SDIRK 2,\n\t"
-                  "7  - SDIRK 3,\n\t"
-                  "8  - CVODE (implicit Adams),\n\t"
-                  "9  - CVODE (implicit BDF),\n\t"
-                  "10 - ARKODE (default explicit),\n\t"
-                  "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default impicit).");
+                  "ODE solver:\n"
+                  "\t 1/11 - CVODE (explicit/implicit),\n"
+                  "\t 2/12 - ARKODE (default explicit/implicit),\n"
+                  "\t 3 - ARKODE (Fehlberg-6-4-5)\n"
+                  "\t 4 - Forward Euler, 5 - RK2, 6 - RK3 SSP, 7 - RK4,\n"
+                  "\t 8 - Backward Euler, 9 - SDIRK23, 10 - SDIRK33.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -179,24 +193,67 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 12)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      }
-      MPI_Finalize();
-      return 1;
-   }
-
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
    //    with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the mesh in serial to increase the resolution. In this example
+   // 4. Define the ODE solver used for time integration. Several
+   // SUNDIALS solvers are available, as well as included both
+   // explicit and implicit MFEM ODE solvers.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver sun_solver; // Used by the implicit SUNDIALS ode solvers.
+   switch (ode_solver_type)
+   {
+      // SUNDIALS solvers
+      case 1:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 11:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF, CV_NEWTON);
+         cvode->SetLinearSolver(sun_solver);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 2:
+      case 3:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 3) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::IMPLICIT);
+         arkode->SetLinearSolver(sun_solver);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Other MFEM explicit methods
+      case 4: ode_solver = new ForwardEulerSolver; break;
+      case 5: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 6: ode_solver = new RK3SSPSolver; break;
+      case 7: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 8:  ode_solver = new BackwardEulerSolver; break;
+      case 9:  ode_solver = new SDIRK23Solver(2); break;
+      case 10: ode_solver = new SDIRK33Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // Since we want to update the diffusion coefficient after every time step,
+   // we need to use the "one-step" mode of the SUNDIALS solvers.
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
+
+   // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
    //    a command-line parameter.
    for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -204,7 +261,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -214,7 +271,7 @@ int main(int argc, char *argv[])
       pmesh->UniformRefinement();
    }
 
-   // 6. Define the vector finite element space representing the current and the
+   // 7. Define the vector finite element space representing the current and the
    //    initial temperature, u_ref.
    H1_FECollection fe_coll(order, dim);
    ParFiniteElementSpace fespace(pmesh, &fe_coll);
@@ -227,14 +284,14 @@ int main(int argc, char *argv[])
 
    ParGridFunction u_gf(&fespace);
 
-   // 7. Set the initial conditions for u. All boundaries are considered
+   // 8. Set the initial conditions for u. All boundaries are considered
    //    natural.
    FunctionCoefficient u_0(InitialTemperature);
    u_gf.ProjectCoefficient(u_0);
    Vector u;
    u_gf.GetTrueDofs(u);
 
-   // 8. Initialize the conduction operator and the VisIt visualization.
+   // 9. Initialize the conduction operator and the VisIt visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
 
    u_gf.SetFromTrueDofs(u);
@@ -293,60 +350,6 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 9. Define the ODE solver used for time integration.
-   double t = 0.0;
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // MFEM explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      // MFEM implicit L-stable methods
-      case 5: ode_solver = new BackwardEulerSolver; break;
-      case 6: ode_solver = new SDIRK23Solver(2); break;
-      case 7: ode_solver = new SDIRK33Solver; break;
-      // CVODE
-      case 8:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      case 9:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      // ARKODE
-      case 10:
-      case 11:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-      case 12:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 8) { ode_solver->Init(oper); }
-
-   // Since we want to update the diffusion coefficient after every time step,
-   // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
-   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
-
    // 10. Perform time-integration (looping over the time iterations, ti, with a
    //     time-step dt).
    if (myid == 0)
@@ -355,6 +358,8 @@ int main(int argc, char *argv[])
    }
    tic_toc.Clear();
    tic_toc.Start();
+   ode_solver->Init(oper);
+   double t = 0.0;
 
    bool last_step = false;
    for (int ti = 1; !last_step; ti++)
@@ -423,7 +428,7 @@ int main(int argc, char *argv[])
 ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
                                        double kap, const Vector &u)
    : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
-     T(NULL),
+     T(NULL), current_dt(0.0),
      M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
 {
    const double rel_tol = 1e-8;
@@ -471,32 +476,30 @@ void ConductionOperator::ImplicitSolve(const double dt,
    // Solve the equation:
    //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
    // for du_dt
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, dt, Kmat);
-   T_solver.SetOperator(*T);
+   if (!T)
+   {
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
    Kmat.Mult(u, z);
    z.Neg();
    T_solver.Mult(z, du_dt);
 }
 
-int ConductionOperator::SUNImplicitSetup(const Vector &x,
-                                         const Vector &fx, int jok, int *jcur,
-                                         double gamma)
+void ConductionOperator::SundialsSolve(const double dt, Vector &b)
 {
-   // Setup the ODE Jacobian T = M + gamma K.
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, gamma, Kmat);
-   T_solver.SetOperator(*T);
-   *jcur = 1;
-   return (0);
-}
-
-int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
-{
-   // Solve the system A x = z => (M - gamma K) x = M b.
+   // Solve the system (M + dt K) y = M b. The result y replaces the input b.
+   if (!T || dt != current_dt)
+   {
+      delete T;
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
    Mmat.Mult(b, z);
-   T_solver.Mult(z, x);
-   return (0);
+   T_solver.Mult(z, b);
 }
 
 void ConductionOperator::SetParameters(const Vector &u)
@@ -516,6 +519,8 @@ void ConductionOperator::SetParameters(const Vector &u)
    K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
    K->Assemble(0); // keep sparsity pattern of M and K the same
    K->FormSystemMatrix(ess_tdof_list, Kmat);
+   delete T;
+   T = NULL; // re-compute T on the next ImplicitSolve or SundialsSolve
 }
 
 ConductionOperator::~ConductionOperator()
@@ -525,6 +530,46 @@ ConductionOperator::~ConductionOperator()
    delete K;
 }
 
+
+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   oper = dynamic_cast<ConductionOperator*>(td_oper);
+   MFEM_VERIFY(oper, "operator is not ConductionOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // oper = static_cast<ConductionOperator*>(td_oper);
+
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   jac_cur = 1;
+
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   oper->SundialsSolve(GetTimeStep(sundials_mem), b);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   return 0;
+}
+
+
 double InitialTemperature(const Vector &x)
 {
    if (x.Norml2() < 0.5)

examples/sundials/ex9.cpp

@@ -4,14 +4,14 @@
 // Compile with: make ex9
 //
 // Sample runs:
-//    ex9 -m ../../data/periodic-segment.mesh -p 0 -r 2 -s 7 -dt 0.005
-//    ex9 -m ../../data/periodic-square.mesh  -p 1 -r 2 -s 8 -dt 0.005 -tf 9
-//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 7 -dt 0.0018 -vs 25
-//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 9 -dt 0.01 -vs 15
-//    ex9 -m ../../data/amr-quad.mesh         -p 1 -r 2 -s 9 -dt 0.002 -tf 9
-//    ex9 -m ../../data/star-q3.mesh          -p 1 -r 2 -s 9 -dt 0.005 -tf 9
-//    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 7 -dt 0.005 -tf 9
-//    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 8 -dt 0.02 -tf 8 -o 2
+//    ex9 -m ../../data/periodic-segment.mesh -p 0 -r 2 -s 11 -dt 0.005
+//    ex9 -m ../../data/periodic-square.mesh  -p 1 -r 2 -s 12 -dt 0.005 -tf 9
+//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 11 -dt 0.0018 -vs 25
+//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 13 -dt 0.01 -vs 15
+//    ex9 -m ../../data/amr-quad.mesh         -p 1 -r 2 -s 13 -dt 0.002 -tf 9
+//    ex9 -m ../../data/star-q3.mesh          -p 1 -r 2 -s 13 -dt 0.005 -tf 9
+//    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 11 -dt 0.005 -tf 9
+//    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 12 -dt 0.02 -tf 8 -o 2
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -109,15 +109,11 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1 - Forward Euler,\n\t"
-                  "2 - RK2 SSP,\n\t"
-                  "3 - RK3 SSP,\n\t"
-                  "4 - RK4,\n\t"
-                  "6 - RK6,\n\t"
-                  "7 - CVODE (adaptive order implicit Adams),\n\t"
-                  "8 - ARKODE default (4th order) explicit,\n\t"
-                  "9 - ARKODE RK8.");
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - CVODE (adaptive order) explicit,\n\t"
+                  "            12 - ARKODE default (4th order) explicit,\n\t"
+                  "            13 - ARKODE RK8.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -139,41 +135,65 @@ int main(int argc, char *argv[])
       args.PrintUsage(cout);
       return 1;
    }
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 9)
-   {
-      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      return 3;
-   }
    args.PrintOptions(cout);
 
    // 2. Read the mesh from the given mesh file. We can handle geometrically
    //    periodic meshes in this code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
 
-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 3. Define the ODE solver used for time integration. Several explicit
+   //    Runge-Kutta methods are available.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      case 11:
+         cvode = new CVODESolver(CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 12:
+      case 13:
+         arkode = new ARKODESolver(ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 13) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement, where 'ref_levels' is a
    //    command-line parameter. If the mesh is of NURBS type, we convert it to
    //    a (piecewise-polynomial) high-order mesh.
    for (int lev = 0; lev < ref_levels; lev++)
    {
-      mesh.UniformRefinement();
+      mesh->UniformRefinement();
    }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
    {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
    }
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
+   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
 
-   // 4. Define the discontinuous DG finite element space of the given
+   // 5. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
    DG_FECollection fec(order, dim);
-   FiniteElementSpace fes(&mesh, &fec);
+   FiniteElementSpace fes(mesh, &fec);
 
    cout << "Number of unknowns: " << fes.GetVSize() << endl;
 
-   // 5. Set up and assemble the bilinear and linear forms corresponding to the
+   // 6. Set up and assemble the bilinear and linear forms corresponding to the
    //    DG discretization. The DGTraceIntegrator involves integrals over mesh
    //    interior faces.
    VectorFunctionCoefficient velocity(dim, velocity_function);
@@ -200,7 +220,7 @@ int main(int argc, char *argv[])
    k.Finalize(skip_zeros);
    b.Assemble();
 
-   // 6. Define the initial conditions, save the corresponding grid function to
+   // 7. Define the initial conditions, save the corresponding grid function to
    //    a file and (optionally) save data in the VisIt format and initialize
    //    GLVis visualization.
    GridFunction u(&fes);
@@ -209,7 +229,7 @@ int main(int argc, char *argv[])
    {
       ofstream omesh("ex9.mesh");
       omesh.precision(precision);
-      mesh.Print(omesh);
+      mesh->Print(omesh);
       ofstream osol("ex9-init.gf");
       osol.precision(precision);
       u.Save(osol);
@@ -223,14 +243,14 @@ int main(int argc, char *argv[])
       if (binary)
       {
 #ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example9", &mesh);
+         dc = new SidreDataCollection("Example9", mesh);
 #else
          MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
 #endif
       }
       else
       {
-         dc = new VisItDataCollection("Example9", &mesh);
+         dc = new VisItDataCollection("Example9", mesh);
          dc->SetPrecision(precision);
       }
       dc->RegisterField("solution", &u);
@@ -255,7 +275,7 @@ int main(int argc, char *argv[])
       else
       {
          sout.precision(precision);
-         sout << "solution\n" << mesh << u;
+         sout << "solution\n" << *mesh << u;
          sout << "pause\n";
          sout << flush;
          cout << "GLVis visualization paused."
@@ -263,46 +283,15 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and define the ODE solver used for time integration.
+   // 8. Define the time-dependent evolution operator describing the ODE
+   //    right-hand side, and perform time-integration (looping over the time
+   //    iterations, ti, with a time-step dt).
    FE_Evolution adv(m.SpMat(), k.SpMat(), b);
 
    double t = 0.0;
    adv.SetTime(t);
+   ode_solver->Init(adv);
 
-   // Create the time integrator
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      case 7:
-         cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(adv);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         cvode->UseSundialsLinearSolver();
-         ode_solver = cvode; break;
-      case 8:
-      case 9:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(adv);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 7) { ode_solver->Init(adv); }
-
-   // 8. Perform time-integration (looping over the time iterations, ti,
-   //    with a time-step dt).
    bool done = false;
    for (int ti = 0; !done; )
    {
@@ -320,7 +309,7 @@ int main(int argc, char *argv[])
 
          if (visualization)
          {
-            sout << "solution\n" << mesh << u << flush;
+            sout << "solution\n" << *mesh << u << flush;
          }
 
          if (visit)

examples/sundials/ex9p.cpp

@@ -4,14 +4,14 @@
 // Compile with: make ex9p
 //
 // Sample runs:
-//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -p 1 -rp 1 -s 7 -dt 0.0025
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh  -p 1 -rp 1 -s 8 -dt 0.0025 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 7 -dt 0.0009 -vs 25
-//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 9 -dt 0.005 -vs 15
-//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh         -p 1 -rp 1 -s 9 -dt 0.001 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/star-q3.mesh          -p 1 -rp 1 -s 9 -dt 0.0025 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 7 -dt 0.0025 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 8 -dt 0.01 -tf 8 -o 2
+//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -p 1 -rp 1 -s 11 -dt 0.0025
+//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh  -p 1 -rp 1 -s 12 -dt 0.0025 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 11 -dt 0.0009 -vs 25
+//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 13 -dt 0.005 -vs 15
+//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh         -p 1 -rp 1 -s 13 -dt 0.001 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/star-q3.mesh          -p 1 -rp 1 -s 13 -dt 0.0025 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 11 -dt 0.0025 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 12 -dt 0.01 -tf 8 -o 2
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -117,15 +117,11 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1 - Forward Euler,\n\t"
-                  "2 - RK2 SSP,\n\t"
-                  "3 - RK3 SSP,\n\t"
-                  "4 - RK4,\n\t"
-                  "6 - RK6,\n\t"
-                  "7 - CVODE (adaptive order implicit Adams),\n\t"
-                  "8 - ARKODE default (4th order) explicit,\n\t"
-                  "9 - ARKODE RK8.");
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - CVODE (adaptive order) explicit,\n\t"
+                  "            12 - ARKODE default (4th order) explicit,\n\t"
+                  "            13 - ARKODE RK8.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
@@ -155,23 +151,47 @@ int main(int argc, char *argv[])
    {
       args.PrintOptions(cout);
    }
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 9)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      }
-      MPI_Finalize();
-      return 3;
-   }
 
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle geometrically periodic meshes in this code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the mesh in serial to increase the resolution. In this example
+   // 4. Define the ODE solver used for time integration. Several explicit
+   //    Runge-Kutta methods are available.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      case 11:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 12:
+      case 13:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 13) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      default:
+         if (myid == 0)
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         MPI_Finalize();
+         return 3;
+   }
+
+   // 5. Refine the mesh in serial to increase the resolution. In this example
    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
    //    a command-line parameter. If the mesh is of NURBS type, we convert it
    //    to a (piecewise-polynomial) high-order mesh.
@@ -185,7 +205,7 @@ int main(int argc, char *argv[])
    }
    mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
 
-   // 5. Define the parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -195,7 +215,7 @@ int main(int argc, char *argv[])
       pmesh->UniformRefinement();
    }
 
-   // 6. Define the parallel discontinuous DG finite element space on the
+   // 7. Define the parallel discontinuous DG finite element space on the
    //    parallel refined mesh of the given polynomial order.
    DG_FECollection fec(order, dim);
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
@@ -206,7 +226,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << global_vSize << endl;
    }
 
-   // 7. Set up and assemble the parallel bilinear and linear forms (and the
+   // 8. Set up and assemble the parallel bilinear and linear forms (and the
    //    parallel hypre matrices) corresponding to the DG discretization. The
    //    DGTraceIntegrator involves integrals over mesh interior faces.
    VectorFunctionCoefficient velocity(dim, velocity_function);
@@ -237,7 +257,7 @@ int main(int argc, char *argv[])
    HypreParMatrix *K = k->ParallelAssemble();
    HypreParVector *B = b->ParallelAssemble();
 
-   // 8. Define the initial conditions, save the corresponding grid function to
+   // 9. Define the initial conditions, save the corresponding grid function to
    //    a file and (optionally) save data in the VisIt format and initialize
    //    GLVis visualization.
    ParGridFunction *u = new ParGridFunction(fes);
@@ -310,46 +330,15 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 9. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and define the ODE solver used for time integration.
+   // 10. Define the time-dependent evolution operator describing the ODE
+   //     right-hand side, and perform time-integration (looping over the time
+   //     iterations, ti, with a time-step dt).
    FE_Evolution adv(*M, *K, *B);
 
    double t = 0.0;
    adv.SetTime(t);
+   ode_solver->Init(adv);
 
-   // Create the time integrator
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      case 7:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
-         cvode->Init(adv);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         cvode->UseSundialsLinearSolver();
-         ode_solver = cvode; break;
-      case 8:
-      case 9:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(adv);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 7) { ode_solver->Init(adv); }
-
-   // 10. Perform time-integration (looping over the time iterations, ti,
-   //     with a time-step dt).
    bool done = false;
    for (int ti = 0; !done; )
    {

examples/sundials/makefile

@@ -60,15 +60,15 @@ PARALLEL_NAME := Parallel SUNDIALS example
 	@$(call mfem-test,$<,, $(SERIAL_NAME))
 
 # Testing: Specific execution options:
-# Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
-EX9_COMMON_ARGS := -m ../../data/periodic-hexagon.mesh -p 0 -s 7
+# Example 9: test explicit CVODE time stepping
+EX9_COMMON_ARGS := -m ../../data/periodic-hexagon.mesh -p 0 -s 11
 EX9_ARGS  := $(EX9_COMMON_ARGS) -r  2 -dt 0.0018 -vs 25
 EX9P_ARGS := $(EX9_COMMON_ARGS) -rp 1 -dt 0.0009 -vs 50
 ex9-test-seq: ex9
 	@$(call mfem-test,$<,, $(SERIAL_NAME),$(EX9_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX9P_ARGS))
-# Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
+# Example 10: test implicit CVODE time stepping
 EX10_COMMON_ARGS := -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10
 EX10_ARGS  := $(EX10_COMMON_ARGS) -r  2
 EX10P_ARGS := $(EX10_COMMON_ARGS) -rp 1

fem/CMakeLists.txt

@@ -11,10 +11,7 @@
 
 set(SRCS
   bilinearform.cpp
-  bilinearform_ext.cpp
   bilininteg.cpp
-  bilininteg_diffusion.cpp
-  bilininteg_mass.cpp
   coefficient.cpp
   datacollection.cpp
   eltrans.cpp
@@ -32,12 +29,10 @@ set(SRCS
   nonlininteg.cpp
   staticcond.cpp
   tmop.cpp
-  tmop_tools.cpp
   )
 
 set(HDRS
   bilinearform.hpp
-  bilinearform_ext.hpp
   bilininteg.hpp
   coefficient.hpp
   datacollection.hpp
@@ -65,7 +60,6 @@ set(HDRS
   tfespace.hpp
   tintrules.hpp
   tmop.hpp
-  tmop_tools.hpp
   )
 
 if (MFEM_USE_SIDRE)

fem/bilinearform.cpp

@@ -12,7 +12,6 @@
 // Implementation of class BilinearForm
 
 #include "fem.hpp"
-#include "../general/device.hpp"
 #include <cmath>
 
 namespace mfem
@@ -63,7 +62,7 @@ void BilinearForm::AllocMat()
    dof_dof.LoseData();
 }
 
-BilinearForm::BilinearForm(FiniteElementSpace * f)
+BilinearForm::BilinearForm (FiniteElementSpace * f)
    : Matrix (f->GetVSize())
 {
    fes = f;
@@ -75,10 +74,6 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
    hybridization = NULL;
    precompute_sparsity = 0;
    diag_policy = DIAG_KEEP;
-
-   assembly = AssemblyLevel::FULL;
-   batch = 1;
-   ext = NULL;
 }
 
 BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
@@ -94,10 +89,6 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
    precompute_sparsity = ps;
    diag_policy = DIAG_KEEP;
 
-   assembly = AssemblyLevel::FULL;
-   batch = 1;
-   ext = NULL;
-
    // Copy the pointers to the integrators
    dbfi = bf->dbfi;
 
@@ -112,44 +103,9 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
    AllocMat();
 }
 
-void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
-{
-   if (ext)
-   {
-      MFEM_ABORT("the assembly level has already been set!");
-   }
-   assembly = assembly_level;
-   switch (assembly)
-   {
-      case AssemblyLevel::FULL:
-         // ext = new FABilinearFormExtension(this);
-         // Use the original BilinearForm implementation for now
-         break;
-      case AssemblyLevel::ELEMENT:
-         mfem_error("Element assembly not supported yet... stay tuned!");
-         // ext = new EABilinearFormExtension(this);
-         break;
-      case AssemblyLevel::PARTIAL:
-         ext = new PABilinearFormExtension(this);
-         break;
-      case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
-         // ext = new MFBilinearFormExtension(this);
-         break;
-      default:
-         mfem_error("Unknown assembly level");
-   }
-}
-
 void BilinearForm::EnableStaticCondensation()
 {
    delete static_cond;
-   if (assembly != AssemblyLevel::FULL)
-   {
-      static_cond = NULL;
-      MFEM_WARNING("Static condensation not supported for this assembly level");
-      return;
-   }
    static_cond = new StaticCondensation(fes);
    if (static_cond->ReducesTrueVSize())
    {
@@ -169,13 +125,6 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                        const Array<int> &ess_tdof_list)
 {
    delete hybridization;
-   if (assembly != AssemblyLevel::FULL)
-   {
-      delete constr_integ;
-      hybridization = NULL;
-      MFEM_WARNING("Hybridization not supported for this assembly level");
-      return;
-   }
    hybridization = new Hybridization(fes, constr_space);
    hybridization->SetConstraintIntegrator(constr_integ);
    hybridization->Init(ess_tdof_list);
@@ -230,9 +179,9 @@ void BilinearForm::Finalize (int skip_zeros)
    if (hybridization) { hybridization->Finalize(); }
 }
 
-void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
+void BilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   dbfi.Append(bfi);
+   dbfi.Append (bfi);
 }
 
 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
@@ -294,33 +243,6 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
    }
 }
 
-void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (bbfi.Size())
-   {
-      const FiniteElement &be = *fes->GetBE(i);
-      ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
-      {
-         bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      fes->GetBdrElementVDofs(i, vdofs);
-      elmat.SetSize(vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void BilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleElementMatrix(i, elmat, vdofs, skip_zeros);
-}
-
 void BilinearForm::AssembleElementMatrix(
    int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
 {
@@ -343,12 +265,6 @@ void BilinearForm::AssembleElementMatrix(
    }
 }
 
-void BilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleBdrElementMatrix(i, elmat, vdofs, skip_zeros);
-}
-
 void BilinearForm::AssembleBdrElementMatrix(
    int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
 {
@@ -371,24 +287,20 @@ void BilinearForm::AssembleBdrElementMatrix(
    }
 }
 
-void BilinearForm::Assemble(int skip_zeros)
+void BilinearForm::Assemble (int skip_zeros)
 {
-   if (ext)
-   {
-      ext->Assemble();
-      return;
-   }
-
    ElementTransformation *eltrans;
    Mesh *mesh = fes -> GetMesh();
    DenseMatrix elmat, *elmat_p;
 
+   int i;
+
    if (mat == NULL)
    {
       AllocMat();
    }
 
-#ifdef MFEM_USE_LEGACY_OPENMP
+#ifdef MFEM_USE_OPENMP
    int free_element_matrices = 0;
    if (!element_matrices)
    {
@@ -399,7 +311,7 @@ void BilinearForm::Assemble(int skip_zeros)
 
    if (dbfi.Size())
    {
-      for (int i = 0; i < fes -> GetNE(); i++)
+      for (i = 0; i < fes -> GetNE(); i++)
       {
          fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
@@ -456,7 +368,7 @@ void BilinearForm::Assemble(int skip_zeros)
          }
       }
 
-      for (int i = 0; i < fes -> GetNBE(); i++)
+      for (i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -494,7 +406,7 @@ void BilinearForm::Assemble(int skip_zeros)
       Array<int> vdofs2;
 
       int nfaces = mesh->GetNumFaces();
-      for (int i = 0; i < nfaces; i++)
+      for (i = 0; i < nfaces; i++)
       {
          tr = mesh -> GetInteriorFaceTransformations (i);
          if (tr != NULL)
@@ -539,7 +451,7 @@ void BilinearForm::Assemble(int skip_zeros)
          }
       }
 
-      for (int i = 0; i < fes -> GetNBE(); i++)
+      for (i = 0; i < fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
@@ -565,7 +477,7 @@ void BilinearForm::Assemble(int skip_zeros)
       }
    }
 
-#ifdef MFEM_USE_LEGACY_OPENMP
+#ifdef MFEM_USE_OPENMP
    if (free_element_matrices)
    {
       FreeElementMatrices();
@@ -608,18 +520,13 @@ void BilinearForm::ConformingAssemble()
    width = mat->Width();
 }
 
-void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
-                                    Vector &b, OperatorHandle &A, Vector &X,
-                                    Vector &B, int copy_interior)
+void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                    Vector &x, Vector &b,
+                                    SparseMatrix &A, Vector &X, Vector &B,
+                                    int copy_interior)
 {
    const SparseMatrix *P = fes->GetConformingProlongation();
 
-   if (ext)
-   {
-      ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-      return;
-   }
-
    FormSystemMatrix(ess_tdof_list, A);
 
    // Transform the system and perform the elimination in B, based on the
@@ -645,8 +552,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       {
          // A, X and B point to the same data as mat, x and b
          EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.NewDataAndSize(x.GetData(), x.Size());
+         B.NewDataAndSize(b.GetData(), b.Size());
          if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
       }
    }
@@ -680,14 +587,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
 }
 
 void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                    OperatorHandle &A)
+                                    SparseMatrix &A)
 {
-   if (ext)
-   {
-      ext->FormSystemMatrix(ess_tdof_list, A);
-      return;
-   }
-
    // Finish the matrix assembly and perform BC elimination, storing the
    // eliminated part of the matrix.
    if (static_cond)
@@ -699,7 +600,7 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
          static_cond->EliminateReducedTrueDofs(diag_policy);
          static_cond->Finalize(); // finalize eliminated part
       }
-      A.Reset(&static_cond->GetMatrix(), false);
+      A.MakeRef(static_cond->GetMatrix());
    }
    else
    {
@@ -713,11 +614,11 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
       }
       if (hybridization)
       {
-         A.Reset(&hybridization->GetMatrix(), false);
+         A.MakeRef(hybridization->GetMatrix());
       }
       else
       {
-         A.Reset(mat, false);
+         A.MakeRef(*mat);
       }
    }
 }
@@ -725,12 +626,6 @@ void BilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 void BilinearForm::RecoverFEMSolution(const Vector &X,
                                       const Vector &b, Vector &x)
 {
-   if (ext)
-   {
-      ext->RecoverFEMSolution(X, b, x);
-      return;
-   }
-
    const SparseMatrix *P = fes->GetConformingProlongation();
    if (!P) // conforming space
    {
@@ -747,10 +642,6 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,
       else
       {
          // X and x point to the same data
-
-         // If the validity flags of X's Memory were changed (e.g. if it was
-         // moved to device memory) then we need to tell x about that.
-         x.SyncMemory(X);
       }
    }
    else // non-conforming space
@@ -796,7 +687,7 @@ void BilinearForm::ComputeElementMatrices()
    DenseMatrix tmp;
    IsoparametricTransformation eltrans;
 
-#ifdef MFEM_USE_LEGACY_OPENMP
+#ifdef MFEM_USE_OPENMP
    #pragma omp parallel for private(tmp,eltrans)
 #endif
    for (int i = 0; i < num_elements; i++)
@@ -823,8 +714,7 @@ void BilinearForm::ComputeElementMatrices()
 }
 
 void BilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                        const Vector &sol, Vector &rhs,
-                                        DiagonalPolicy dpolicy)
+                                        const Vector &sol, Vector &rhs, DiagonalPolicy dpolicy)
 {
    Array<int> ess_dofs, conf_ess_dofs;
    fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
@@ -998,8 +888,6 @@ void BilinearForm::Update(FiniteElementSpace *nfes)
    }
 
    height = width = fes->GetVSize();
-
-   if (ext) { ext->Update(); }
 }
 
 void BilinearForm::SetDiagonalPolicy(DiagonalPolicy policy)
@@ -1023,8 +911,6 @@ BilinearForm::~BilinearForm()
       for (k=0; k < fbfi.Size(); k++) { delete fbfi[k]; }
       for (k=0; k < bfbfi.Size(); k++) { delete bfbfi[k]; }
    }
-
-   delete ext;
 }
 
 
@@ -1049,13 +935,9 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    extern_bfs = 1;
 
    // Copy the pointers to the integrators
-   dbfi = mbf->dbfi;
-   bbfi = mbf->bbfi;
-   tfbfi = mbf->tfbfi;
-   btfbfi = mbf->btfbfi;
-
-   bbfi_marker = mbf->bbfi_marker;
-   btfbfi_marker = mbf->btfbfi_marker;
+   dom = mbf->dom;
+   bdr = mbf->bdr;
+   skt = mbf->skt;
 }
 
 double & MixedBilinearForm::Elem (int i, int j)
@@ -1109,42 +991,22 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
 
 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   dbfi.Append (bfi);
+   dom.Append (bfi);
 }
 
 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                                               Array<int> &bdr_marker)
-{
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   bdr.Append (bfi);
 }
 
 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   tfbfi.Append (bfi);
-}
-
-void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi)
-{
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
-                                                  Array<int> &bdr_marker)
-{
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(&bdr_marker);
+   skt.Append (bfi);
 }
 
 void MixedBilinearForm::Assemble (int skip_zeros)
 {
+   int i, k;
    Array<int> tr_vdofs, te_vdofs;
    ElementTransformation *eltrans;
    DenseMatrix elemmat;
@@ -1156,75 +1018,48 @@ void MixedBilinearForm::Assemble (int skip_zeros)
       mat = new SparseMatrix(height, width);
    }
 
-   if (dbfi.Size())
+   if (dom.Size())
    {
-      for (int i = 0; i < test_fes -> GetNE(); i++)
+      for (i = 0; i < test_fes -> GetNE(); i++)
       {
          trial_fes -> GetElementVDofs (i, tr_vdofs);
          test_fes  -> GetElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetElementTransformation (i);
-         for (int k = 0; k < dbfi.Size(); k++)
+         for (k = 0; k < dom.Size(); k++)
          {
-            dbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                               *test_fes  -> GetFE(i),
-                                               *eltrans, elemmat);
+            dom[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                              *test_fes  -> GetFE(i),
+                                              *eltrans, elemmat);
             mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
 
-   if (bbfi.Size())
+   if (bdr.Size())
    {
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (i = 0; i < test_fes -> GetNBE(); i++)
       {
-         if (bbfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *bbfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < test_fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
          trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
          test_fes  -> GetBdrElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetBdrElementTransformation (i);
-         for (int k = 0; k < bbfi.Size(); k++)
+         for (k = 0; k < bdr.Size(); k++)
          {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-            bbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
-                                               *test_fes  -> GetBE(i),
-                                               *eltrans, elemmat);
+            bdr[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                              *test_fes  -> GetBE(i),
+                                              *eltrans, elemmat);
             mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
 
-   if (tfbfi.Size())
+   if (skt.Size())
    {
       FaceElementTransformations *ftr;
       Array<int> te_vdofs2;
       const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
 
       int nfaces = mesh->GetNumFaces();
-      for (int i = 0; i < nfaces; i++)
+      for (i = 0; i < nfaces; i++)
       {
          ftr = mesh->GetFaceElementTransformations(i);
          trial_fes->GetFaceVDofs(i, tr_vdofs);
@@ -1244,70 +1079,14 @@ void MixedBilinearForm::Assemble (int skip_zeros)
             // want to actually make a fake element.
             test_fe2 = test_fe1;
          }
-         for (int k = 0; k < tfbfi.Size(); k++)
+         for (int k = 0; k < skt.Size(); k++)
          {
-            tfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                         *ftr, elemmat);
+            skt[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
+                                       *ftr, elemmat);
             mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
-
-   if (btfbfi.Size())
-   {
-      FaceElementTransformations *ftr;
-      Array<int> te_vdofs2;
-      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
-
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < btfbfi.Size(); k++)
-      {
-         if (btfbfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *btfbfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary trace face integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < trial_fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         ftr = mesh->GetBdrFaceTransformations(i);
-         if (ftr)
-         {
-            trial_fes->GetFaceVDofs(i, tr_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
-            trial_face_fe = trial_fes->GetFaceElement(i);
-            test_fe1 = test_fes->GetFE(ftr->Elem1No);
-            // The test_fe2 object is really a dummy and not used on the
-            // boundaries, but we can't dereference a NULL pointer, and we don't
-            // want to actually make a fake element.
-            test_fe2 = test_fe1;
-            for (int k = 0; k < btfbfi.Size(); k++)
-            {
-               if (btfbfi_marker[k] &&
-                   (*btfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-               btfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                             *ftr, elemmat);
-               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
-            }
-         }
-      }
-   }
 }
 
 void MixedBilinearForm::ConformingAssemble()
@@ -1336,93 +1115,8 @@ void MixedBilinearForm::ConformingAssemble()
    width = mat->Width();
 }
 
-
-void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (dbfi.Size())
-   {
-      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
-      const FiniteElement &test_fe = *test_fes->GetFE(i);
-      ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
-      {
-         dbfi[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      trial_fes->GetElementVDofs(i, trial_vdofs);
-      test_fes->GetElementVDofs(i, test_vdofs);
-      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (bbfi.Size())
-   {
-      const FiniteElement &trial_be = *trial_fes->GetBE(i);
-      const FiniteElement &test_be = *test_fes->GetBE(i);
-      ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
-      {
-         bbfi[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-      test_fes->GetBdrElementVDofs(i, test_vdofs);
-      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleElementMatrix(i, elmat, trial_vdofs, test_vdofs, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
-{
-   trial_fes->GetElementVDofs(i, trial_vdofs);
-   test_fes->GetElementVDofs(i, test_vdofs);
-   if (mat == NULL)
-   {
-      mat = new SparseMatrix(height, width);
-   }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleBdrElementMatrix(i, elmat, trial_vdofs, test_vdofs, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
-{
-   trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-   test_fes->GetBdrElementVDofs(i, test_vdofs);
-   if (mat == NULL)
-   {
-      mat = new SparseMatrix(height, width);
-   }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
-}
-
 void MixedBilinearForm::EliminateTrialDofs (
-   const Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
+   Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
 {
    int i, j, k;
    Array<int> tr_vdofs, cols_marker (trial_fes -> GetVSize());
@@ -1445,12 +1139,12 @@ void MixedBilinearForm::EliminateTrialDofs (
 }
 
 void MixedBilinearForm::EliminateEssentialBCFromTrialDofs (
-   const Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
+   Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
 {
    mat -> EliminateCols (marked_vdofs, &sol, &rhs);
 }
 
-void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
+void MixedBilinearForm::EliminateTestDofs (Array<int> &bdr_attr_is_ess)
 {
    int i, j, k;
    Array<int> te_vdofs;
@@ -1484,10 +1178,9 @@ MixedBilinearForm::~MixedBilinearForm()
    if (!extern_bfs)
    {
       int i;
-      for (i = 0; i < dbfi.Size(); i++) { delete dbfi[i]; }
-      for (i = 0; i < bbfi.Size(); i++) { delete bbfi[i]; }
-      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
-      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
+      for (i = 0; i < dom.Size(); i++) { delete dom[i]; }
+      for (i = 0; i < bdr.Size(); i++) { delete bdr[i]; }
+      for (i = 0; i < skt.Size(); i++) { delete skt[i]; }
    }
 }
 
@@ -1504,7 +1197,7 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
       mat = new SparseMatrix(height, width);
    }
 
-   if (dbfi.Size() > 0)
+   if (dom.Size() > 0)
    {
       for (int i = 0; i < test_fes->GetNE(); i++)
       {
@@ -1514,17 +1207,17 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
          dom_fe = trial_fes->GetFE(i);
          ran_fe = test_fes->GetFE(i);
 
-         dbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < dbfi.Size(); j++)
+         dom[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < dom.Size(); j++)
          {
-            dbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            dom[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
             totelmat += elmat;
          }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
       }
    }
 
-   if (tfbfi.Size())
+   if (skt.Size())
    {
       const int nfaces = test_fes->GetMesh()->GetNumFaces();
       for (int i = 0; i < nfaces; i++)
@@ -1535,10 +1228,10 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
          dom_fe = trial_fes->GetFaceElement(i);
          ran_fe = test_fes->GetFaceElement(i);
 
-         tfbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < tfbfi.Size(); j++)
+         skt[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < skt.Size(); j++)
          {
-            tfbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            skt[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
             totelmat += elmat;
          }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);

fem/bilinearform.hpp

@@ -18,32 +18,12 @@
 #include "gridfunc.hpp"
 #include "linearform.hpp"
 #include "bilininteg.hpp"
-#include "bilinearform_ext.hpp"
 #include "staticcond.hpp"
 #include "hybridization.hpp"
 
 namespace mfem
 {
 
-/// Enumeration defining the assembly level for bilinear and nonlinear form
-/// classes derived from Operator.
-enum class AssemblyLevel
-{
-   /// Fully assembled form, i.e. a global sparse matrix in MFEM, Hypre or PETSC
-   /// format.
-   FULL,
-   /// Form assembled at element level, which computes and stores dense element
-   /// matrices.
-   ELEMENT,
-   /// Partially-assembled form, which computes and stores data only at
-   /// quadrature points.
-   PARTIAL,
-   /// "Matrix-free" form that computes all of its action on-the-fly without any
-   /// substantial storage.
-   NONE,
-};
-
-
 /** Class for bilinear form - "Matrix" with associated FE space and
     BLFIntegrators. */
 class BilinearForm : public Matrix
@@ -58,14 +38,6 @@ protected:
    /// FE space on which the form lives. Not owned.
    FiniteElementSpace *fes;
 
-   /// The form assembly level (full, partial, etc.)
-   AssemblyLevel assembly;
-   /// Element batch size used in the form action (1, 8, num_elems, etc.)
-   int batch;
-   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
-       Partial Assembly (PA), or Matrix Free assembly (MF). */
-   BilinearFormExtension *ext;
-
    /// Indicates the Mesh::sequence corresponding to the current state of the
    /// BilinearForm.
    long sequence;
@@ -117,9 +89,6 @@ protected:
       static_cond = NULL; hybridization = NULL;
       precompute_sparsity = 0;
       diag_policy = DIAG_KEEP;
-      assembly = AssemblyLevel::FULL;
-      batch = 1;
-      ext = NULL;
    }
 
 private:
@@ -149,10 +118,6 @@ public:
    /// Get the size of the BilinearForm as a square matrix.
    int Size() const { return height; }
 
-   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
-   /** This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level);
-
    /** Enable the use of static condensation. For details see the description
        for class StaticCondensation in fem/staticcond.hpp This method should be
        called before assembly. If the number of unknowns after static
@@ -326,12 +291,11 @@ public:
    virtual const Operator *GetRestriction() const
    { return fes->GetConformingRestriction(); }
 
-   /** @brief Form the linear system A X = B, corresponding to this bilinear
-       form and the linear form @a b(.). */
-   /** This method applies any necessary transformations to the linear system
-       such as: eliminating boundary conditions; applying conforming constraints
-       for non-conforming AMR; parallel assembly; static condensation;
-       hybridization.
+   /// Form a linear system, A X = B.
+   /** Form the linear system A X = B, corresponding to the current bilinear
+       form and b(.), by applying any necessary transformations such as:
+       eliminating boundary conditions; applying conforming constraints for
+       non-conforming AMR; static condensation; hybridization.
 
        The GridFunction-size vector @a x must contain the essential b.c. The
        BilinearForm and the LinearForm-size vector @a b must be assembled.
@@ -352,52 +316,12 @@ public:
 
        NOTE: If there are no transformations, @a X simply reuses the data of
              @a x. */
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
-                                 Vector &b, OperatorHandle &A, Vector &X,
-                                 Vector &B, int copy_interior = 0);
-
-   /** @brief Form the linear system A X = B, corresponding to this bilinear
-       form and the linear form @a b(.). */
-   /** Version of the method FormLinearSystem() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
-   template <typename OpType>
    void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
-                         OpType &A, Vector &X, Vector &B,
-                         int copy_interior = 0)
-   {
-      OperatorHandle Ah;
-      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
-   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A);
+                         SparseMatrix &A, Vector &X, Vector &B,
+                         int copy_interior = 0);
 
    /// Form the linear system matrix A, see FormLinearSystem() for details.
-   /** Version of the method FormSystemMatrix() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
-   template <typename OpType>
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
-   {
-      OperatorHandle Ah;
-      FormSystemMatrix(ess_tdof_list, Ah);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, SparseMatrix &A);
 
    /// Recover the solution of a linear system formed with FormLinearSystem().
    /** Call this method after solving a linear system constructed using the
@@ -413,49 +337,9 @@ public:
    void FreeElementMatrices()
    { delete element_matrices; element_matrices = NULL; }
 
-   /// Compute the element matrix of the given element
-   /** The element matrix is computed by calling the domain integrators
-       or the one stored internally by a prior call of ComputeElementMatrices()
-       is returned when available.
-   */
    void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Compute the boundary element matrix of the given boundary element
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The flag @a skip_zeros skips the zero
-       elements of the matrix, unless they are breaking the symmetry of
-       the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              int skip_zeros = 1);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The vdofs of the element are returned
-       in @a vdofs. The flag @a skip_zeros skips the zero elements of the
-       matrix, unless they are breaking the symmetry of the system matrix.
-   */
    void AssembleElementMatrix(int i, const DenseMatrix &elmat,
                               Array<int> &vdofs, int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The vdofs of the element
-       are returned in @a vdofs. The flag @a skip_zeros skips the zero elements
-       of the matrix, unless they are breaking the symmetry of the system matrix.
-   */
    void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
                                  Array<int> &vdofs, int skip_zeros = 1);
 
@@ -529,7 +413,6 @@ public:
    virtual ~BilinearForm();
 };
 
-
 /**
    Class for assembling of bilinear forms `a(u,v)` defined on different
    trial and test spaces. The assembled matrix `A` is such that
@@ -553,26 +436,16 @@ protected:
    FiniteElementSpace *trial_fes, ///< Not owned
                       *test_fes;  ///< Not owned
 
-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #dom, #bdr, and
+       #skt are owned by another MixedBilinearForm. */
    int extern_bfs;
 
    /// Domain integrators.
-   Array<BilinearFormIntegrator*> dbfi;
-
+   Array<BilinearFormIntegrator*> dom;
    /// Boundary integrators.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker;///< Entries are not owned.
-
+   Array<BilinearFormIntegrator*> bdr;
    /// Trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> tfbfi;
-
-   /// Boundary trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> btfbfi;
-   Array<Array<int>*>             btfbfi_marker;///< Entries are not owned.
-
-   DenseMatrix elemmat;
-   Array<int>  trial_vdofs, test_vdofs;
+   Array<BilinearFormIntegrator*> skt;
 
 private:
    /// Copy construction is not supported; body is undefined.
@@ -636,10 +509,6 @@ public:
    /// Adds a boundary integrator. Assumes ownership of @a bfi.
    void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);
 
-   /// Adds a boundary integrator. Assumes ownership of @a bfi.
-   void AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                               Array<int> &bdr_marker);
-
    /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
 
        This type of integrator assembles terms over all faces of the mesh using
@@ -647,32 +516,14 @@ public:
        test space. */
    void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi);
 
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi,
-                                   Array<int> &bdr_marker);
-
    /// Access all integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &dom; }
 
    /// Access all integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
-   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &bdr; }
 
    /// Access all integrators added with AddTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetTFBFI() { return &tfbfi; }
-
-   /// Access all integrators added with AddBdrTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBTFBFI() { return &btfbfi; }
-   /** @brief Access all boundary markers added with AddBdrTraceFaceIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }
+   Array<BilinearFormIntegrator*> *GetTFBFI() { return &skt; }
 
    void operator=(const double a) { *mat = a; }
 
@@ -685,59 +536,13 @@ public:
        MixedBilinearForm becomes an operator on the conforming FE spaces. */
    void ConformingAssemble();
 
-   /// Compute the element matrix of the given element
-   void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Compute the boundary element matrix of the given boundary element
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The flag @a skip_zeros skips the zero
-       elements of the matrix, unless they are breaking the symmetry of
-       the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              int skip_zeros = 1);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The vdofs of the element are returned
-       in @a trial_vdofs and @a test_vdofs. The flag @a skip_zeros skips
-       the zero elements of the matrix, unless they are breaking the symmetry
-       of the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              Array<int> &trial_vdofs, Array<int> &test_vdofs,
-                              int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The vdofs of the element
-       are returned in @a trial_vdofs and @a test_vdofs. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 Array<int> &trial_vdofs, Array<int> &test_vdofs,
-                                 int skip_zeros = 1);
-
-   void EliminateTrialDofs(const Array<int> &bdr_attr_is_ess,
+   void EliminateTrialDofs(Array<int> &bdr_attr_is_ess,
                            const Vector &sol, Vector &rhs);
 
-   void EliminateEssentialBCFromTrialDofs(const Array<int> &marked_vdofs,
+   void EliminateEssentialBCFromTrialDofs(Array<int> &marked_vdofs,
                                           const Vector &sol, Vector &rhs);
 
-   virtual void EliminateTestDofs(const Array<int> &bdr_attr_is_ess);
+   virtual void EliminateTestDofs(Array<int> &bdr_attr_is_ess);
 
    void Update();
 
@@ -802,7 +607,7 @@ public:
    { AddTraceFaceIntegrator(di); }
 
    /// Access all interpolators added with AddDomainInterpolator().
-   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDI() { return &dom; }
 
    /** @brief Construct the internal matrix representation of the discrete
        linear operator. */

fem/bilinearform_ext.cpp

@@ -1,151 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-// Implementations of classes FABilinearFormExtension, EABilinearFormExtension,
-// PABilinearFormExtension and MFBilinearFormExtension.
-
-#include "../general/forall.hpp"
-#include "bilinearform.hpp"
-
-namespace mfem
-{
-
-BilinearFormExtension::BilinearFormExtension(BilinearForm *form)
-   : Operator(form->Size()), a(form)
-{
-   // empty
-}
-
-const Operator *BilinearFormExtension::GetProlongation() const
-{
-   return a->GetProlongation();
-}
-
-const Operator *BilinearFormExtension::GetRestriction() const
-{
-   return a->GetRestriction();
-}
-
-
-// Data and methods for partially-assembled bilinear forms
-PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
-   : BilinearFormExtension(form),
-     trialFes(a->FESpace()), testFes(a->FESpace())
-{
-   elem_restrict_lex = trialFes->GetElementRestriction(
-                          ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_lex)
-   {
-      localX.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
-   }
-}
-
-void PABilinearFormExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssemblePA(*a->FESpace());
-   }
-}
-
-void PABilinearFormExtension::Update()
-{
-   FiniteElementSpace *fes = a->FESpace();
-   height = width = fes->GetVSize();
-   trialFes = fes;
-   testFes = fes;
-   elem_restrict_lex = trialFes->GetElementRestriction(
-                          ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_lex)
-   {
-      localX.SetSize(elem_restrict_lex->Height());
-      localY.SetSize(elem_restrict_lex->Height());
-   }
-}
-
-void PABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                               OperatorHandle &A)
-{
-   const Operator* trialP = trialFes->GetProlongationMatrix();
-   const Operator* testP  = testFes->GetProlongationMatrix();
-   Operator *rap = this;
-   if (trialP) { rap = new RAPOperator(*testP, *this, *trialP); }
-   const bool own_A = (rap!=this);
-   A.Reset(new ConstrainedOperator(rap, ess_tdof_list, own_A));
-}
-
-void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                               Vector &x, Vector &b,
-                                               OperatorHandle &A,
-                                               Vector &X, Vector &B,
-                                               int copy_interior)
-{
-   Operator *oper;
-   Operator::FormLinearSystem(ess_tdof_list, x, b, oper, X, B, copy_interior);
-   A.Reset(oper); // A will own oper
-}
-
-void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (elem_restrict_lex)
-   {
-      elem_restrict_lex->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(localX, localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(x, y);
-      }
-   }
-}
-
-void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-   if (elem_restrict_lex)
-   {
-      elem_restrict_lex->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposePA(localX, localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true);
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposePA(x, y);
-      }
-   }
-}
-
-} // namespace mfem

fem/bilinearform_ext.hpp

@@ -1,139 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_BILINEARFORM_EXT
-#define MFEM_BILINEARFORM_EXT
-
-#include "../config/config.hpp"
-#include "fespace.hpp"
-#include "../general/device.hpp"
-
-namespace mfem
-{
-
-class BilinearForm;
-
-
-/** @brief Class extending the BilinearForm class to support the different
-    AssemblyLevel%s. */
-class BilinearFormExtension : public Operator
-{
-protected:
-   BilinearForm *a; ///< Not owned
-
-public:
-   BilinearFormExtension(BilinearForm *form);
-
-   virtual MemoryClass GetMemoryClass() const
-   { return Device::GetMemoryClass(); }
-
-   /// Get the finite element space prolongation matrix
-   virtual const Operator *GetProlongation() const;
-
-   /// Get the finite element space restriction matrix
-   virtual const Operator *GetRestriction() const;
-
-   virtual void Assemble() = 0;
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A) = 0;
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
-                                 Vector &x, Vector &b,
-                                 OperatorHandle &A, Vector &X, Vector &B,
-                                 int copy_interior = 0) = 0;
-   virtual void Update() = 0;
-};
-
-/// Data and methods for fully-assembled bilinear forms
-class FABilinearFormExtension : public BilinearFormExtension
-{
-public:
-   FABilinearFormExtension(BilinearForm *form)
-      : BilinearFormExtension(form) { }
-
-   /// TODO
-   void Assemble() {}
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
-   void FormLinearSystem(const Array<int> &ess_tdof_list,
-                         Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) {}
-   void Mult(const Vector &x, Vector &y) const {}
-   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
-   ~FABilinearFormExtension() {}
-};
-
-/// Data and methods for element-assembled bilinear forms
-class EABilinearFormExtension : public BilinearFormExtension
-{
-public:
-   EABilinearFormExtension(BilinearForm *form)
-      : BilinearFormExtension(form) { }
-
-   /// TODO
-   void Assemble() {}
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
-   void FormLinearSystem(const Array<int> &ess_tdof_list,
-                         Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) {}
-   void Mult(const Vector &x, Vector &y) const {}
-   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
-   ~EABilinearFormExtension() {}
-};
-
-/// Data and methods for partially-assembled bilinear forms
-class PABilinearFormExtension : public BilinearFormExtension
-{
-protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
-   mutable Vector localX, localY;
-   const Operator *elem_restrict_lex; // Not owned
-
-public:
-   PABilinearFormExtension(BilinearForm*);
-
-   void Assemble();
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
-   void FormLinearSystem(const Array<int> &ess_tdof_list,
-                         Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
-
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-   void Update();
-};
-
-/// Data and methods for matrix-free bilinear forms
-class MFBilinearFormExtension : public BilinearFormExtension
-{
-public:
-   MFBilinearFormExtension(BilinearForm *form)
-      : BilinearFormExtension(form) { }
-
-   /// TODO
-   void Assemble() {}
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
-   void FormLinearSystem(const Array<int> &ess_tdof_list,
-                         Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) {}
-   void Mult(const Vector &x, Vector &y) const {}
-   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
-   ~MFBilinearFormExtension() {}
-};
-
-}
-
-#endif

fem/bilininteg.cpp

@@ -20,24 +20,6 @@ using namespace std;
 namespace mfem
 {
 
-void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
-{
-   mfem_error ("BilinearFormIntegrator::Assemble (...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::MultAssembled (...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose (...)\n"
-               "   is not implemented for this class.");
-}
-
 void BilinearFormIntegrator::AssembleElementMatrix (
    const FiniteElement &el, ElementTransformation &Trans,
    DenseMatrix &elmat )
@@ -379,7 +361,6 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
    }
 }
 
-
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -399,7 +380,29 @@ void DiffusionIntegrator::AssembleElementMatrix
 #endif
    elmat.SetSize(nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -455,7 +458,28 @@ void DiffusionIntegrator::AssembleElementMatrix2(
 #endif
    elmat.SetSize(te_nd, tr_nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (trial_fe.Space() == FunctionSpace::Pk)
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
+      }
+      else
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() + dim - 1;
+      }
+
+      if (trial_fe.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      }
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -510,7 +534,29 @@ void DiffusionIntegrator::AssembleElementVector(
 
    elvect.SetSize(nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }
 
    elvect = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -670,27 +716,6 @@ double DiffusionIntegrator::ComputeFluxEnergy
    return energy;
 }
 
-const IntegrationRule &DiffusionIntegrator::GetRule(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe)
-{
-   int order;
-   if (trial_fe.Space() == FunctionSpace::Pk)
-   {
-      order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
-   }
-   else
-   {
-      // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
-      order = trial_fe.GetOrder() + test_fe.GetOrder() + trial_fe.GetDim() - 1;
-   }
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
 
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
@@ -706,7 +731,21 @@ void MassIntegrator::AssembleElementMatrix
    elmat.SetSize(nd);
    shape.SetSize(nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      // int order = 2 * el.GetOrder();
+      int order = 2 * el.GetOrder() + Trans.OrderW();
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -741,8 +780,13 @@ void MassIntegrator::AssembleElementMatrix2(
    shape.SetSize(tr_nd);
    te_shape.SetSize(te_nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule :
-                               &GetRule(trial_fe, test_fe, Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -763,20 +807,6 @@ void MassIntegrator::AssembleElementMatrix2(
    }
 }
 
-const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
-                                               const FiniteElement &test_fe,
-                                               ElementTransformation &Trans)
-{
-   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
-   const int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
 
 void BoundaryMassIntegrator::AssembleFaceMatrix(
    const FiniteElement &el1, const FiniteElement &el2,
@@ -848,7 +878,7 @@ void ConvectionIntegrator::AssembleElementMatrix(
       ir = &IntRules.Get(el.GetGeomType(), order);
    }
 
-   Q->Eval(Q_ir, Trans, *ir);
+   Q.Eval(Q_ir, Trans, *ir);
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -889,7 +919,7 @@ void GroupConvectionIntegrator::AssembleElementMatrix(
       ir = &IntRules.Get(el.GetGeomType(), order);
    }
 
-   Q->Eval(Q_nodal, Trans, el.GetNodes()); // sets the size of Q_nodal
+   Q.Eval(Q_nodal, Trans, el.GetNodes()); // sets the size of Q_nodal
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -1370,7 +1400,7 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
          dshapedxi(l) = dshapedxt(l,xi);
       }
 
-      shape *= Q->Eval(Trans,ip) * det * ip.weight;
+      shape *= Q.Eval(Trans,ip) * det * ip.weight;
       AddMultVWt (shape, dshapedxi, elmat);
    }
 }
@@ -3216,7 +3246,7 @@ ScalarProductInterpolator::AssembleElementMatrix2(const FiniteElement &dom_fe,
                                                   ElementTransformation &Trans,
                                                   DenseMatrix &elmat)
 {
-   internal::ShapeCoefficient dom_shape_coeff(*Q, dom_fe);
+   internal::ShapeCoefficient dom_shape_coeff(Q, dom_fe);
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 
@@ -3251,7 +3281,7 @@ ScalarVectorProductInterpolator::AssembleElementMatrix2(
       }
    };
 
-   VShapeCoefficient dom_shape_coeff(*Q, dom_fe, Trans.GetSpaceDim());
+   VShapeCoefficient dom_shape_coeff(Q, dom_fe, Trans.GetSpaceDim());
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 
@@ -3289,7 +3319,7 @@ VectorScalarProductInterpolator::AssembleElementMatrix2(
       }
    };
 
-   VecShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   VecShapeCoefficient dom_shape_coeff(VQ, dom_fe);
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 
@@ -3336,11 +3366,11 @@ VectorCrossProductInterpolator::AssembleElementMatrix2(
       }
    };
 
-   VCrossVShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   VCrossVShapeCoefficient dom_shape_coeff(VQ, dom_fe);
 
    if (ran_fe.GetRangeType() == FiniteElement::SCALAR)
    {
-      elmat.SetSize(ran_fe.GetDof()*VQ->GetVDim(),dom_fe.GetDof());
+      elmat.SetSize(ran_fe.GetDof()*VQ.GetVDim(),dom_fe.GetDof());
    }
    else
    {
@@ -3389,7 +3419,7 @@ VectorInnerProductInterpolator::AssembleElementMatrix2(
    ElementTransformation &Trans,
    DenseMatrix &elmat)
 {
-   internal::VDotVShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   internal::VDotVShapeCoefficient dom_shape_coeff(VQ, dom_fe);
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 

fem/bilininteg.hpp

@@ -14,7 +14,6 @@
 
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
-#include "fespace.hpp"
 
 namespace mfem
 {
@@ -23,45 +22,10 @@ namespace mfem
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
 protected:
-   BilinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : NonlinearFormIntegrator(ir) { }
+   BilinearFormIntegrator(const IntegrationRule *ir = NULL) :
+      NonlinearFormIntegrator(ir) { }
 
 public:
-   // TODO: add support for other assembly levels (in addition to PA) and their
-   // actions.
-
-   // TODO: for mixed meshes the quadrature rules to be used by methods like
-   // AssemblePA() can be given as a QuadratureSpace, e.g. using a new method:
-   // SetQuadratureSpace().
-
-   // TODO: the methods for the various assembly levels make sense even in the
-   // base class NonlinearFormIntegrator, except that not all assembly levels
-   // make sense for the action of the nonlinear operator (but they all make
-   // sense for its Jacobian).
-
-   /// Method defining partial assembly.
-   /** The result of the partial assembly is stored internally so that it can be
-       used later in the methods AddMultPA() and AddMultTransposePA(). */
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-
-   /// Method for partially assembled action.
-   /** Perform the action of integrator on the input @a x and add the result to
-       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
-       the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-
-   /// Method for partially assembled transposed action.
-   /** Perform the transpose action of integrator on the input @a x and add the
-       result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
-       represent the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
    /// Given a particular Finite Element computes the element matrix elmat.
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
@@ -309,10 +273,10 @@ protected:
                                       Vector & shape)
    { trial_fe.CalcPhysShape(Trans, shape); }
 
-   Coefficient *Q;
-
 private:
 
+   Coefficient *Q;
+
 #ifndef MFEM_THREAD_SAFE
    Vector test_shape;
    Vector trial_shape;
@@ -383,13 +347,13 @@ protected:
                                       DenseMatrix & shape)
    { trial_fe.CalcVShape(Trans, shape); }
 
+private:
+
    Coefficient *Q;
    VectorCoefficient *VQ;
    VectorCoefficient *DQ;
    MatrixCoefficient *MQ;
 
-private:
-
 #ifndef MFEM_THREAD_SAFE
    Vector V;
    Vector D;
@@ -464,12 +428,12 @@ protected:
                                  Vector & shape)
    { scalar_fe.CalcPhysShape(Trans, shape); }
 
+private:
+
    VectorCoefficient *VQ;
    bool transpose;
    bool cross_2d;  // In 2D use a cross product rather than a dot product
 
-private:
-
 #ifndef MFEM_THREAD_SAFE
    Vector V;
    DenseMatrix vshape;
@@ -1662,34 +1626,24 @@ protected:
     can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-   MatrixCoefficient *MQ;
-
 private:
    Vector vec, pointflux, shape;
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape, dshapedxt, invdfdx, mq;
    DenseMatrix te_dshape, te_dshapedxt;
 #endif
-
-   // PA extension
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, dofs1D, quad1D;
-   Vector pa_data;
+   Coefficient *Q;
+   MatrixCoefficient *MQ;
 
 public:
    /// Construct a diffusion integrator with coefficient Q = 1
-   DiffusionIntegrator() { Q = NULL; MQ = NULL; maps = NULL; geom = NULL; }
+   DiffusionIntegrator() { Q = NULL; MQ = NULL; }
 
    /// Construct a diffusion integrator with a scalar coefficient q
-   DiffusionIntegrator(Coefficient &q)
-      : Q(&q) { MQ = NULL; maps = NULL; geom = NULL; }
+   DiffusionIntegrator (Coefficient &q) : Q(&q) { MQ = NULL; }
 
    /// Construct a diffusion integrator with a matrix coefficient q
-   DiffusionIntegrator(MatrixCoefficient &q)
-      : MQ(&q) { Q = NULL; maps = NULL; geom = NULL; }
+   DiffusionIntegrator (MatrixCoefficient &q) : MQ(&q) { Q = NULL; }
 
    /** Given a particular Finite Element
        computes the element stiffness matrix elmat. */
@@ -1716,13 +1670,6 @@ public:
    virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                     ElementTransformation &Trans,
                                     Vector &flux, Vector *d_energy = NULL);
-
-   virtual void AssemblePA(const FiniteElementSpace&);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe);
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1733,19 +1680,13 @@ protected:
    Vector shape, te_shape;
 #endif
    Coefficient *Q;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
 
 public:
    MassIntegrator(const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir) { Q = NULL; maps = NULL; geom = NULL; }
-
+      : BilinearFormIntegrator(ir) { Q = NULL; }
    /// Construct a mass integrator with coefficient q
    MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(&q) { maps = NULL; geom = NULL; }
+      : BilinearFormIntegrator(ir), Q(&q) { }
 
    /** Given a particular Finite Element
        computes the element mass matrix elmat. */
@@ -1756,14 +1697,6 @@ public:
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
-
-   virtual void AssemblePA(const FiniteElementSpace&);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
 };
 
 class BoundaryMassIntegrator : public MassIntegrator
@@ -1782,19 +1715,17 @@ public:
 /// alpha (q . grad u, v)
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
-protected:
-   VectorCoefficient *Q;
-   double alpha;
-
 private:
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape, adjJ, Q_ir;
    Vector shape, vec2, BdFidxT;
 #endif
+   VectorCoefficient &Q;
+   double alpha;
 
 public:
    ConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
-      : Q(&q) { alpha = a; }
+      : Q(q) { alpha = a; }
    virtual void AssembleElementMatrix(const FiniteElement &,
                                       ElementTransformation &,
                                       DenseMatrix &);
@@ -1803,17 +1734,15 @@ public:
 /// alpha (q . grad u, v) using the "group" FE discretization
 class GroupConvectionIntegrator : public BilinearFormIntegrator
 {
-protected:
-   VectorCoefficient *Q;
-   double alpha;
-
 private:
    DenseMatrix dshape, adjJ, Q_nodal, grad;
    Vector shape;
+   VectorCoefficient &Q;
+   double alpha;
 
 public:
    GroupConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
-      : Q(&q) { alpha = a; }
+      : Q(q) { alpha = a; }
    virtual void AssembleElementMatrix(const FiniteElement &,
                                       ElementTransformation &,
                                       DenseMatrix &);
@@ -1829,17 +1758,16 @@ private:
    Vector shape, te_shape, vec;
    DenseMatrix partelmat;
    DenseMatrix mcoeff;
-   int Q_order;
-
-protected:
    Coefficient *Q;
    VectorCoefficient *VQ;
    MatrixCoefficient *MQ;
 
+   int Q_order;
+
 public:
    /// Construct an integrator with coefficient 1.0
    VectorMassIntegrator()
-      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
+      : vdim(-1), Q(NULL), VQ(NULL), MQ(NULL), Q_order(0) { }
    /** Construct an integrator with scalar coefficient q.
        If possible, save memory by using a scalar integrator since
        the resulting matrix is block diagonal with the same diagonal
@@ -1878,14 +1806,11 @@ public:
     does NOT depend on the ElementTransformation Trans. */
 class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
    Vector divshape, shape;
 #endif
-
 public:
    VectorFEDivergenceIntegrator() { Q = NULL; }
    VectorFEDivergenceIntegrator(Coefficient &q) { Q = &q; }
@@ -1903,17 +1828,14 @@ public:
     This is equivalent to a weak divergence of the Nedelec basis functions. */
 class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape;
    DenseMatrix dshapedxt;
    DenseMatrix vshape;
    DenseMatrix invdfdx;
 #endif
-
 public:
    VectorFEWeakDivergenceIntegrator() { Q = NULL; }
    VectorFEWeakDivergenceIntegrator(Coefficient &q) { Q = &q; }
@@ -1930,16 +1852,13 @@ public:
     test spaces are switched, assembles the form (u, curl v). */
 class VectorFECurlIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix curlshapeTrial;
    DenseMatrix vshapeTest;
    DenseMatrix curlshapeTrial_dFT;
 #endif
-
 public:
    VectorFECurlIntegrator() { Q = NULL; }
    VectorFECurlIntegrator(Coefficient &q) { Q = &q; }
@@ -1952,19 +1871,17 @@ public:
                                        DenseMatrix &elmat);
 };
 
+
 /// Class for integrating (Q D_i(u), v); u and v are scalars
 class DerivativeIntegrator : public BilinearFormIntegrator
 {
-protected:
-   Coefficient* Q;
-
 private:
+   Coefficient & Q;
    int xi;
    DenseMatrix dshape, dshapedxt, invdfdx;
    Vector shape, dshapedxi;
-
 public:
-   DerivativeIntegrator(Coefficient &q, int i) : Q(&q), xi(i) { }
+   DerivativeIntegrator(Coefficient &q, int i) : Q(q), xi(i) { }
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat)
@@ -1984,8 +1901,6 @@ private:
    DenseMatrix curlshape, curlshape_dFt, M;
    DenseMatrix vshape, projcurl;
 #endif
-
-protected:
    Coefficient *Q;
    MatrixCoefficient *MQ;
 
@@ -2019,8 +1934,6 @@ private:
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape_hat, dshape, curlshape, Jadj, grad_hat, grad;
 #endif
-
-protected:
    Coefficient *Q;
 
 public:
@@ -2042,6 +1955,9 @@ public:
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
+   Coefficient *Q;
+   VectorCoefficient *VQ;
+   MatrixCoefficient *MQ;
    void Init(Coefficient *q, VectorCoefficient *vq, MatrixCoefficient *mq)
    { Q = q; VQ = vq; MQ = mq; }
 
@@ -2053,11 +1969,6 @@ private:
    DenseMatrix trial_vshape;
 #endif
 
-protected:
-   Coefficient *Q;
-   VectorCoefficient *VQ;
-   MatrixCoefficient *MQ;
-
 public:
    VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
    VectorFEMassIntegrator(Coefficient *_q) { Init(_q, NULL, NULL); }
@@ -2080,10 +1991,9 @@ public:
     scalar FE space; p is also in a (different) scalar FE space.  */
 class VectorDivergenceIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *Q;
 
-private:
    Vector shape;
    Vector divshape;
    DenseMatrix dshape;
@@ -2104,10 +2014,9 @@ public:
 /// (Q div u, div v) for RT elements
 class DivDivIntegrator: public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *Q;
 
-private:
 #ifndef MFEM_THREAD_SAFE
    Vector divshape;
 #endif
@@ -2129,10 +2038,9 @@ public:
     diffusion matrix in each diagonal block. */
 class VectorDiffusionIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *Q;
 
-private:
    DenseMatrix Jinv;
    DenseMatrix dshape;
    DenseMatrix gshape;
@@ -2157,11 +2065,10 @@ public:
     using multiple copies of a scalar FE space. */
 class ElasticityIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    double q_lambda, q_mu;
    Coefficient *lambda, *mu;
 
-private:
 #ifndef MFEM_THREAD_SAFE
    Vector shape;
    DenseMatrix dshape, gshape, pelmat;
@@ -2218,12 +2125,11 @@ public:
     points. */
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *rho;
    VectorCoefficient *u;
    double alpha, beta;
 
-private:
    Vector shape1, shape2;
 
 public:
@@ -2510,7 +2416,7 @@ public:
 class ScalarProductInterpolator : public DiscreteInterpolator
 {
 public:
-   ScalarProductInterpolator(Coefficient & sc) : Q(&sc) { }
+   ScalarProductInterpolator(Coefficient & sc) : Q(sc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
@@ -2518,7 +2424,7 @@ public:
                                        DenseMatrix &elmat);
 
 protected:
-   Coefficient *Q;
+   Coefficient &Q;
 };
 
 /** Interpolator of a scalar coefficient multiplied by a vector field onto
@@ -2528,14 +2434,14 @@ class ScalarVectorProductInterpolator : public DiscreteInterpolator
 {
 public:
    ScalarVectorProductInterpolator(Coefficient & sc)
-      : Q(&sc) { }
+      : Q(sc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   Coefficient *Q;
+   Coefficient &Q;
 };
 
 /** Interpolator of a vector coefficient multiplied by a scalar field onto
@@ -2545,14 +2451,14 @@ class VectorScalarProductInterpolator : public DiscreteInterpolator
 {
 public:
    VectorScalarProductInterpolator(VectorCoefficient & vc)
-      : VQ(&vc) { }
+      : VQ(vc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };
 
 /** Interpolator of the cross product between a vector coefficient and an
@@ -2562,14 +2468,14 @@ class VectorCrossProductInterpolator : public DiscreteInterpolator
 {
 public:
    VectorCrossProductInterpolator(VectorCoefficient & vc)
-      : VQ(&vc) { }
+      : VQ(vc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &nd_fe,
                                        const FiniteElement &rt_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };
 
 /** Interpolator of the inner product between a vector coefficient and an
@@ -2578,14 +2484,14 @@ protected:
 class VectorInnerProductInterpolator : public DiscreteInterpolator
 {
 public:
-   VectorInnerProductInterpolator(VectorCoefficient & vc) : VQ(&vc) { }
+   VectorInnerProductInterpolator(VectorCoefficient & vc) : VQ(vc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &rt_fe,
                                        const FiniteElement &l2_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };
 
 }

fem/bilininteg_mass.cpp

@@ -1,801 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Mass Integrator
-
-// PA Mass Assemble kernel
-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   // Assuming the same element type
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
-   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
-                                    GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, Device::GetMemoryType());
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.Write(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
-   if (dim==2)
-   {
-      const int NE = ne;
-      const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
-      auto w = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
-      auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      MFEM_FORALL(e, NE,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double J11 = J(q,0,0,e);
-            const double J12 = J(q,1,0,e);
-            const double J21 = J(q,0,1,e);
-            const double J22 = J(q,1,1,e);
-            const double detJ = (J11*J22)-(J21*J12);
-            const double coeff = const_c ? C(0,0) : C(q,e);
-            v(q,e) =  w[q] * coeff * detJ;
-         }
-      });
-   }
-   if (dim==3)
-   {
-      const int NE = ne;
-      const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
-      auto W = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
-      auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
-      auto v = Reshape(pa_data.Write(), NQ,NE);
-      MFEM_FORALL(e, NE,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e);
-            const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e);
-            const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e);
-            const double detJ = J11 * (J22 * J33 - J32 * J23) -
-            /* */               J21 * (J12 * J33 - J32 * J13) +
-            /* */               J31 * (J12 * J23 - J22 * J13);
-            const double coeff = const_c ? C(0,0) : C(q,e);
-            v(q,e) = W[q] * coeff * detJ;
-         }
-      });
-   }
-}
-
-#ifdef MFEM_USE_OCCA
-// OCCA PA Mass Apply 2D kernel
-static void OccaPAMassApply2D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &op,
-                              const Vector &x,
-                              Vector &y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_op = OccaMemoryRead(op.GetMemory(), op.Size());
-   const occa::memory o_x = OccaMemoryRead(x.GetMemory(), x.Size());
-   occa::memory o_y = OccaMemoryReadWrite(y.GetMemory(), y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply2D_cpu;
-      if (OccaMassApply2D_cpu.find(id) == OccaMassApply2D_cpu.end())
-      {
-         const occa::kernel MassApply2D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_CPU", props);
-         OccaMassApply2D_cpu.emplace(id, MassApply2D_CPU);
-      }
-      OccaMassApply2D_cpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply2D_gpu;
-      if (OccaMassApply2D_gpu.find(id) == OccaMassApply2D_gpu.end())
-      {
-         const occa::kernel MassApply2D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_GPU", props);
-         OccaMassApply2D_gpu.emplace(id, MassApply2D_GPU);
-      }
-      OccaMassApply2D_gpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-}
-
-// OCCA PA Mass Apply 3D kernel
-static void OccaPAMassApply3D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &op,
-                              const Vector &x,
-                              Vector &y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_op = OccaMemoryRead(op.GetMemory(), op.Size());
-   const occa::memory o_x = OccaMemoryRead(x.GetMemory(), x.Size());
-   occa::memory o_y = OccaMemoryReadWrite(y.GetMemory(), y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply3D_cpu;
-      if (OccaMassApply3D_cpu.find(id) == OccaMassApply3D_cpu.end())
-      {
-         const occa::kernel MassApply3D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_CPU", props);
-         OccaMassApply3D_cpu.emplace(id, MassApply3D_CPU);
-      }
-      OccaMassApply3D_cpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply3D_gpu;
-      if (OccaMassApply3D_gpu.find(id) == OccaMassApply3D_gpu.end())
-      {
-         const occa::kernel MassApply3D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_GPU", props);
-         OccaMassApply3D_gpu.emplace(id, MassApply3D_GPU);
-      }
-      OccaMassApply3D_gpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-}
-#endif // MFEM_USE_OCCA
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void PAMassApply2D(const int NE,
-                          const Array<double> &B_,
-                          const Array<double> &Bt_,
-                          const Vector &op_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(B_.Read(), Q1D, D1D);
-   auto Bt = Reshape(Bt_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         double sol_x[max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            sol_x[qy] = 0.0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double s = x(dx,dy,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] += B(qx,dx)* s;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double d2q = B(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] += d2q * sol_x[qx];
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] *= op(qx,qy,e);
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double sol_x[max_D1D];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] = 0.0;
-         }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double s = sol_xy[qy][qx];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] += Bt(dx,qx) * s;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            const double q2d = Bt(dy,qy);
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               y(dx,dy,e) += q2d * sol_x[dx];
-            }
-         }
-      }
-   });
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0,
-         const int T_NBZ = 0>
-static void SmemPAMassApply2D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double BBt[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) BBt;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
-      MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
-      double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
-      double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
-      double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
-      double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            X[dy][dx] = x(dx,dy,e);
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double dq = 0.0;
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               dq += X[dy][dx] * B[qx][dx];
-            }
-            DQ[dy][qx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double qq = 0.0;
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               qq += DQ[dy][qx] * B[qy][dy];
-            }
-            QQ[qy][qx] = qq * op(qx, qy, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[d][q] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dq = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dq += QQ[qy][qx] * Bt[dx][qx];
-            }
-            QD[qy][dx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dd = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               dd += (QD[qy][dx] * Bt[dy][qy]);
-            }
-            y(dx, dy, e) += dd;
-         }
-      }
-   });
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void PAMassApply3D(const int NE,
-                          const Array<double> &B_,
-                          const Array<double> &Bt_,
-                          const Vector &op_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(B_.Read(), Q1D, D1D);
-   auto Bt = Reshape(Bt_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         double sol_xy[max_Q1D][max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double sol_x[max_Q1D];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] = 0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double s = x(dx,dy,dz,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_x[qx] += B(qx,dx) * s;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy = B(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xy[qy][qx] += wy * sol_x[qx];
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const double wz = B(qz,dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-               }
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] *= op(qx,qy,qz,e);
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         double sol_xy[max_D1D][max_D1D];
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_xy[dy][dx] = 0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double sol_x[max_D1D];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] = 0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double s = sol_xyz[qz][qy][qx];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_x[dx] += Bt(dx,qx) * s;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double wy = Bt(dy,qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_xy[dy][dx] += wy * sol_x[dx];
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            const double wz = Bt(dz,qz);
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
-               }
-            }
-         }
-      }
-   });
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void SmemPAMassApply3D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= M1D, "");
-   MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sDQ[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sDQ;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
-      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-      double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
-      double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
-      double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
-      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
-      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
-      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               X[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  u += X[dz][dy][dx] * B[qx][dx];
-               }
-               DDQ[dz][dy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-               DQQ[dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u += DQQ[dz][qy][qx] * B[qz][dz];
-               }
-               QQQ[qz][qy][qx] = u * op(qx,qy,qz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[d][q] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += QQQ[qz][qy][qx] * Bt[dx][qx];
-               }
-               QQD[qz][qy][dx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-               QDD[qz][dy][dx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-               y(dx,dy,dz,e) += u;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassApply(const int dim,
-                        const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const Array<double> &B,
-                        const Array<double> &Bt,
-                        const Vector &op,
-                        const Vector &x,
-                        Vector &y)
-{
-#ifdef MFEM_USE_OCCA
-   if (DeviceCanUseOcca())
-   {
-      if (dim == 2)
-      {
-         OccaPAMassApply2D(D1D, Q1D, NE, B, Bt, op, x, y);
-         return;
-      }
-      if (dim == 3)
-      {
-         OccaPAMassApply3D(D1D, Q1D, NE, B, Bt, op, x, y);
-         return;
-      }
-      MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
-   }
-#endif // MFEM_USE_OCCA
-   if (dim == 2)
-   {
-      switch ((D1D << 4) | Q1D)
-      {
-         case 0x22: return SmemPAMassApply2D<2,2,16>(NE, B, Bt, op, x, y);
-         case 0x33: return SmemPAMassApply2D<3,3,16>(NE, B, Bt, op, x, y);
-         case 0x44: return SmemPAMassApply2D<4,4,8>(NE, B, Bt, op, x, y);
-         case 0x55: return SmemPAMassApply2D<5,5,8>(NE, B, Bt, op, x, y);
-         case 0x66: return SmemPAMassApply2D<6,6,4>(NE, B, Bt, op, x, y);
-         case 0x77: return SmemPAMassApply2D<7,7,4>(NE, B, Bt, op, x, y);
-         case 0x88: return SmemPAMassApply2D<8,8,2>(NE, B, Bt, op, x, y);
-         case 0x99: return SmemPAMassApply2D<9,9,2>(NE, B, Bt, op, x, y);
-         default:   return PAMassApply2D(NE, B, Bt, op, x, y, D1D, Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4) | Q1D)
-      {
-         case 0x23: return SmemPAMassApply3D<2,3>(NE, B, Bt, op, x, y);
-         case 0x34: return SmemPAMassApply3D<3,4>(NE, B, Bt, op, x, y);
-         case 0x45: return SmemPAMassApply3D<4,5>(NE, B, Bt, op, x, y);
-         case 0x56: return SmemPAMassApply3D<5,6>(NE, B, Bt, op, x, y);
-         case 0x67: return SmemPAMassApply3D<6,7>(NE, B, Bt, op, x, y);
-         case 0x78: return SmemPAMassApply3D<7,8>(NE, B, Bt, op, x, y);
-         case 0x89: return SmemPAMassApply3D<8,9>(NE, B, Bt, op, x, y);
-         default:   return PAMassApply3D(NE, B, Bt, op, x, y, D1D, Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
-}
-
-} // namespace mfem

fem/coefficient.cpp

@@ -125,27 +125,19 @@ void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
 }
 
 VectorArrayCoefficient::VectorArrayCoefficient (int dim)
-   : VectorCoefficient(dim), Coeff(dim), ownCoeff(dim)
+   : VectorCoefficient(dim), Coeff(dim)
 {
    for (int i = 0; i < dim; i++)
    {
       Coeff[i] = NULL;
-      ownCoeff[i] = true;
    }
 }
 
-void VectorArrayCoefficient::Set(int i, Coefficient *c, bool own)
-{
-   if (ownCoeff[i]) { delete Coeff[i]; }
-   Coeff[i] = c;
-   ownCoeff[i] = own;
-}
-
 VectorArrayCoefficient::~VectorArrayCoefficient()
 {
    for (int i = 0; i < vdim; i++)
    {
-      if (ownCoeff[i]) { delete Coeff[i]; }
+      delete Coeff[i];
    }
 }
 
@@ -317,26 +309,17 @@ MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
    : MatrixCoefficient (dim)
 {
    Coeff.SetSize(height*width);
-   ownCoeff.SetSize(height*width);
    for (int i = 0; i < (height*width); i++)
    {
       Coeff[i] = NULL;
-      ownCoeff[i] = true;
    }
 }
 
-void MatrixArrayCoefficient::Set(int i, int j, Coefficient * c, bool own)
-{
-   if (ownCoeff[i*width+j]) { delete Coeff[i*width+j]; }
-   Coeff[i*width+j] = c;
-   ownCoeff[i*width+j] = own;
-}
-
 MatrixArrayCoefficient::~MatrixArrayCoefficient ()
 {
    for (int i=0; i < height*width; i++)
    {
-      if (ownCoeff[i]) { delete Coeff[i]; }
+      delete Coeff[i];
    }
 }
 

fem/coefficient.hpp

@@ -96,7 +96,7 @@ public:
    { constants.SetSize(c.Size()); constants=c; }
 
    /// Update constants
-   void UpdateConstants(Vector &c) { constants.SetSize(c.Size()); constants=c; }
+   void UpdateConstants(Vector &c) {constants.SetSize(c.Size()); constants=c;}
 
    /// Member function to access or modify the value of the i-th constant
    double &operator()(int i) { return constants(i-1); }
@@ -112,7 +112,6 @@ public:
                        const IntegrationPoint &ip);
 };
 
-
 /// class for C-function coefficient
 class FunctionCoefficient : public Coefficient
 {
@@ -369,7 +368,6 @@ class VectorArrayCoefficient : public VectorCoefficient
 {
 private:
    Array<Coefficient*> Coeff;
-   Array<bool> ownCoeff;
 
 public:
    /// Construct vector of dim coefficients.
@@ -381,7 +379,7 @@ public:
    Coefficient **GetCoeffs() { return Coeff; }
 
    /// Sets coefficient in the vector.
-   void Set(int i, Coefficient *c, bool own=true);
+   void Set(int i, Coefficient *c) { delete Coeff[i]; Coeff[i] = c; }
 
    /// Evaluates i'th component of the vector.
    double Eval(int i, ElementTransformation &T, const IntegrationPoint &ip)
@@ -450,7 +448,6 @@ public:
    void SetGridFunction(GridFunction *gf);
    GridFunction * GetGridFunction() const { return GridFunc; }
 
-   using VectorCoefficient::Eval;
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
 
@@ -501,13 +498,9 @@ public:
    void SetDeltaCoefficient(const DeltaCoefficient& _d) { d = _d; }
    /// Return the associated scalar DeltaCoefficient.
    DeltaCoefficient& GetDeltaCoefficient() { return d; }
-
-   void SetScale(double s) { d.SetScale(s); }
    void SetDirection(const Vector& _d);
 
-   void SetDeltaCenter(const Vector& center) { d.SetDeltaCenter(center); }
    void GetDeltaCenter(Vector& center) { d.GetDeltaCenter(center); }
-
    /** @brief Return the specified direction vector multiplied by the value
        returned by DeltaCoefficient::EvalDelta() of the associated scalar
        DeltaCoefficient. */
@@ -633,7 +626,6 @@ class MatrixArrayCoefficient : public MatrixCoefficient
 {
 private:
    Array<Coefficient *> Coeff;
-   Array<bool> ownCoeff;
 
 public:
 
@@ -641,7 +633,7 @@ public:
 
    Coefficient* GetCoeff (int i, int j) { return Coeff[i*width+j]; }
 
-   void Set(int i, int j, Coefficient * c, bool own=true);
+   void Set(int i, int j, Coefficient * c) { delete Coeff[i*width+j]; Coeff[i*width+j] = c; }
 
    double Eval(int i, int j, ElementTransformation &T, const IntegrationPoint &ip)
    { return Coeff[i*width+j] ? Coeff[i*width+j] -> Eval(T, ip, GetTime()) : 0.0; }
@@ -798,7 +790,6 @@ public:
    /// Evaluate the coefficient
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
-   using VectorCoefficient::Eval;
 };
 
 /// Vector coefficient defined as a product of a scalar and a vector
@@ -813,7 +804,6 @@ public:
 
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
-   using VectorCoefficient::Eval;
 };
 
 /// Vector coefficient defined as a cross product of two vectors
@@ -831,7 +821,6 @@ public:
 
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
-   using VectorCoefficient::Eval;
 };
 
 /// Vector coefficient defined as a matrix vector product
@@ -849,7 +838,6 @@ public:
 
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
-   using VectorCoefficient::Eval;
 };
 
 /// Matrix coefficient defined as the identity of dimension d

fem/datacollection.cpp

@@ -108,7 +108,6 @@ DataCollection::DataCollection(const std::string& collection_name, Mesh *mesh_)
    precision = precision_default;
    pad_digits_cycle = pad_digits_rank = pad_digits_default;
    format = SERIAL_FORMAT; // use serial mesh format
-   compression = false;
    error = NO_ERROR;
 }
 
@@ -162,14 +161,6 @@ void DataCollection::SetFormat(int fmt)
    format = fmt;
 }
 
-void DataCollection::SetCompression(bool comp)
-{
-   compression = comp;
-#ifdef MFEM_USE_GZSTREAM
-   MFEM_ASSERT(!compression, "GZStream not enabled in MFEM build.");
-#endif
-}
-
 void DataCollection::SetPrefixPath(const std::string& prefix)
 {
    if (!prefix.empty())
@@ -228,8 +219,7 @@ void DataCollection::SaveMesh()
    }
 
    std::string mesh_name = GetMeshFileName();
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream mesh_file(mesh_name.c_str(), mode);
+   std::ofstream mesh_file(mesh_name.c_str());
    mesh_file.precision(precision);
 #ifdef MFEM_USE_MPI
    const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
@@ -277,9 +267,7 @@ const
 
 void DataCollection::SaveOneField(const FieldMapIterator &it)
 {
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream field_file(GetFieldFileName(it->first).c_str(), mode);
-
+   std::ofstream field_file(GetFieldFileName(it->first).c_str());
    field_file.precision(precision);
    (it->second)->Save(field_file);
    if (!field_file)
@@ -291,8 +279,7 @@ void DataCollection::SaveOneField(const FieldMapIterator &it)
 
 void DataCollection::SaveOneQField(const QFieldMapIterator &it)
 {
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream q_field_file(GetFieldFileName(it->first).c_str(), mode);
+   std::ofstream q_field_file(GetFieldFileName(it->first).c_str());
    q_field_file.precision(precision);
    (it->second)->Save(q_field_file);
    if (!q_field_file)
@@ -589,7 +576,7 @@ void VisItDataCollection::LoadFields()
         it != field_info_map.end(); ++it)
    {
       std::string fname = path_left + it->first + path_right;
-      ifgzstream file(fname.c_str());
+      std::ifstream file(fname.c_str());
       // TODO: in parallel, check for errors on all processors
       if (!file)
       {

fem/datacollection.hpp

@@ -205,7 +205,6 @@ protected:
 
    /// Output mesh format: see the #Format enumeration
    int format;
-   bool compression;
 
    /// Should the collection delete its mesh and fields
    bool own_data;
@@ -347,9 +346,6 @@ public:
        validation. */
    virtual void SetFormat(int fmt);
 
-   /// Set the flag for use of gz compressed files
-   void SetCompression(bool comp);
-
    /// Set the path where the DataCollection will be saved.
    void SetPrefixPath(const std::string &prefix);
 

fem/fe.cpp

@@ -203,22 +203,6 @@ void FiniteElement::CalcPhysDShape(ElementTransformation &Trans,
    Mult(vshape, Trans.InverseJacobian(), dshape);
 }
 
-const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &,
-                                             DofToQuad::Mode) const
-{
-   mfem_error("FiniteElement::GetDofToQuad(...) is not implemented for "
-              "this element!");
-   return *dof2quad_array[0]; // suppress a warning
-}
-
-FiniteElement::~FiniteElement()
-{
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      delete dof2quad_array[i];
-   }
-}
-
 
 void ScalarFiniteElement::NodalLocalInterpolation (
    ElementTransformation &Trans, DenseMatrix &I,
@@ -294,95 +278,6 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
    }
 }
 
-const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
-                                                   DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = Dof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*Dof);
-   d2q->Bt.SetSize(Dof*nqpt);
-   d2q->G.SetSize(nqpt*Dim*Dof);
-   d2q->Gt.SetSize(Dof*nqpt*Dim);
-#ifdef MFEM_THREAD_SAFE
-   Vector c_shape(Dof);
-   DenseMatrix vshape(Dof, Dim);
-#endif
-   for (int i = 0; i < nqpt; i++)
-   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
-      CalcShape(ip, c_shape);
-      for (int j = 0; j < Dof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+Dof*i] = c_shape(j);
-      }
-      CalcDShape(ip, vshape);
-      for (int d = 0; d < Dim; d++)
-      {
-         for (int j = 0; j < Dof; j++)
-         {
-            d2q->G[i+nqpt*(d+Dim*j)] = d2q->Gt[j+Dof*(i+nqpt*d)] = vshape(j,d);
-         }
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-
-// protected method
-const DofToQuad &ScalarFiniteElement::GetTensorDofToQuad(
-   const TensorBasisElement &tb,
-   const IntegrationRule &ir, DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const Poly_1D::Basis &basis_1d = tb.GetBasis1D();
-   const int ndof = Order + 1;
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/Dim) + 0.5);
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis_1d.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-
 
 void NodalFiniteElement::ProjectCurl_2D(
    const FiniteElement &fe, ElementTransformation &Trans,
@@ -9635,7 +9530,6 @@ void L2_TetrahedronElement::ProjectDelta(int vertex, Vector &dofs) const
             const IntegrationPoint &ip = Nodes.IntPoint(i);
             dofs[i] = pow(ip.y, Order);
          }
-         break;
       case 3:
          for (int i = 0; i < Dof; i++)
          {
@@ -11966,10 +11860,6 @@ Linear3DFiniteElement TetrahedronFE;
 // Object declared in mesh/wedge.hpp.
 // Defined here to ensure it is constructed after 'poly1d' and before
 // 'Geometries'.
-// TODO: define as thread_local to prevent race conditions in GLVis, because
-// there is no "LinearWedgeFiniteElement" and WedgeFE is in turn used from two
-// different threads for different things in GLVis. We also don't want to turn
-// MFEM_THREAD_SAFE on globally. (See PR #731)
 H1_WedgeElement WedgeFE(1);
 
 // Object declared in geom.hpp.

fem/fe.hpp

@@ -116,92 +116,7 @@ public:
    }
 };
 
-
-/** @brief Structure representing the matrices/tensors needed to evaluate (in
-    reference space) the values, gradients, divergences, or curls of a
-    FiniteElement at a the quadrature points of a given IntegrationRule. */
-/** Object of this type are typically created and owned by the respective
-    FiniteElement object. */
-class DofToQuad
-{
-public:
-   /// The FiniteElement that created and owns this object.
-   /** This pointer is not owned. */
-   const class FiniteElement *FE;
-
-   /** @brief IntegrationRule that defines the quadrature points at which the
-       basis functions of the #FE are evaluated. */
-   /** This pointer is not owned. */
-   const IntegrationRule *IntRule;
-
-   /// Type of data stored in the arrays #B, #Bt, #G, and #Gt.
-   enum Mode
-   {
-      /** @brief Full multidimensional representation which does not use tensor
-          product structure. The ordering of the degrees of freedom is as
-          defined by #FE */
-      FULL,
-
-      /** @brief Tensor product representation using 1D matrices/tensors with
-          dimensions using 1D number of quadrature points and degrees of
-          freedom. */
-      /** When representing a vector-valued FiniteElement, two DofToQuad objects
-          are used to describe the "closed" and "open" 1D basis functions
-          (TODO). */
-      TENSOR
-   };
-
-   /// Describes the contents of the #B, #Bt, #G, and #Gt arrays, see #Mode.
-   Mode mode;
-
-   /** @brief Number of degrees of freedom = number of basis functions. When
-       #mode is TENSOR, this is the 1D number. */
-   int ndof;
-
-   /** @brief Number of quadrature points. When #mode is TENSOR, this is the 1D
-       number. */
-   int nqpt;
-
-   /// Basis functions evaluated at quadrature points.
-   /** The storage layout is column-major with dimensions:
-       - #nqpt x #ndof, for scalar elements, or
-       - #nqpt x dim x #ndof, for vector elements, (TODO)
-
-       where
-
-       - dim = dimension of the finite element reference space when #mode is
-         FULL, and dim = 1 when #mode is TENSOR. */
-   Array<double> B;
-
-   /// Transpose of #B.
-   /** The storage layout is column-major with dimensions:
-       - #ndof x #nqpt, for scalar elements, or
-       - #ndof x #nqpt x dim, for vector elements (TODO). */
-   Array<double> Bt;
-
-   /** @brief Gradients/divergences/curls of basis functions evaluated at
-       quadrature points. */
-   /** The storage layout is column-major with dimensions:
-       - #nqpt x dim x #ndof, for scalar elements, or
-       - #nqpt x #ndof, for H(div) vector elements (TODO), or
-       - #nqpt x cdim x #ndof, for H(curl) vector elements (TODO),
-
-       where
-
-       - dim = dimension of the finite element reference space when #mode is
-         FULL, and 1 when #mode is TENSOR,
-       - cdim = 1/1/3 in 1D/2D/3D, respectively, when #mode is FULL, and cdim =
-         1 when #mode is TENSOR. */
-   Array<double> G;
-
-   /// Transpose of #G.
-   /** The storage layout is column-major with dimensions:
-       - #ndof x #nqpt x dim, for scalar elements, or
-       - #ndof x #nqpt, for H(div) vector elements (TODO), or
-       - #ndof x #nqpt x cdim, for H(curl) vector elements (TODO). */
-   Array<double> Gt;
-};
-
+// Base and derived classes for finite elements
 
 /// Describes the space on each element
 class FunctionSpace
@@ -221,10 +136,6 @@ class VectorCoefficient;
 class MatrixCoefficient;
 class KnotVector;
 
-
-// Base and derived classes for finite elements
-
-
 /// Abstract class for Finite Elements
 class FiniteElement
 {
@@ -241,10 +152,6 @@ protected:
 #ifndef MFEM_THREAD_SAFE
    mutable DenseMatrix vshape; // Dof x Dim
 #endif
-   /// Container for all DofToQuad objects created by the FiniteElement.
-   /** Multiple DofToQuad objects may be needed when different quadrature rules
-       or different DofToQuad::Mode are used. */
-   mutable Array<DofToQuad*> dof2quad_array;
 
 public:
    /// Enumeration for RangeType and DerivRangeType
@@ -510,13 +417,7 @@ public:
                            ElementTransformation &Trans,
                            DenseMatrix &div) const;
 
-   /** Return a DofToQuad structure corresponding to the given IntegrationRule
-       using the given DofToQuad::Mode. */
-   /** See the documentation for DofToQuad for more details. */
-   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                         DofToQuad::Mode mode) const;
-
-   virtual ~FiniteElement();
+   virtual ~FiniteElement () { }
 
    static bool IsClosedType(int b_type)
    {
@@ -563,10 +464,6 @@ protected:
       return static_cast<const ScalarFiniteElement &>(fe);
    }
 
-   const DofToQuad &GetTensorDofToQuad(const class TensorBasisElement &tb,
-                                       const IntegrationRule &ir,
-                                       DofToQuad::Mode mode) const;
-
 public:
    ScalarFiniteElement(int D, Geometry::Type G, int Do, int O,
                        int F = FunctionSpace::Pk)
@@ -597,9 +494,6 @@ public:
    void ScalarLocalInterpolation(ElementTransformation &Trans,
                                  DenseMatrix &I,
                                  const ScalarFiniteElement &fine_fe) const;
-
-   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                         DofToQuad::Mode mode) const;
 };
 
 class NodalFiniteElement : public ScalarFiniteElement
@@ -1856,14 +1750,6 @@ class NodalTensorFiniteElement : public NodalFiniteElement,
 public:
    NodalTensorFiniteElement(const int dims, const int p, const int btype,
                             const DofMapType dmtype);
-
-   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const
-   {
-      return (mode == DofToQuad::FULL) ?
-             ScalarFiniteElement::GetDofToQuad(ir, mode) :
-             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
-   }
 };
 
 class PositiveTensorFiniteElement : public PositiveFiniteElement,
@@ -1872,14 +1758,6 @@ class PositiveTensorFiniteElement : public PositiveFiniteElement,
 public:
    PositiveTensorFiniteElement(const int dims, const int p,
                                const DofMapType dmtype);
-
-   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const
-   {
-      return (mode == DofToQuad::FULL) ?
-             ScalarFiniteElement::GetDofToQuad(ir, mode) :
-             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
-   }
 };
 
 class H1_SegmentElement : public NodalTensorFiniteElement

fem/fem.hpp

@@ -31,7 +31,6 @@
 #include "estimators.hpp"
 #include "staticcond.hpp"
 #include "tmop.hpp"
-#include "tmop_tools.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/fespace.hpp

@@ -59,33 +59,12 @@ Ordering::Map<Ordering::byVDIM>(int ndofs, int vdim, int dof, int vd)
 }
 
 
-/// Constants describing the possible orderings of the DOFs in one element.
-enum class ElementDofOrdering
-{
-   /// Native ordering as defined by the FiniteElement.
-   /** This ordering can be used by tensor-product elements when the
-       interpolation from the DOFs to quadrature points does not use the
-       tensor-product structure. */
-   NATIVE,
-   /// Lexicographic ordering for tensor-product FiniteElements.
-   /** This ordering can be used only with tensor-product elements. */
-   LEXICOGRAPHIC
-};
-
-
-// Forward declarations
 class NURBSExtension;
-class BilinearFormIntegrator;
-class QuadratureSpace;
-class QuadratureInterpolator;
-
 
 /** @brief Class FiniteElementSpace - responsible for providing FEM view of the
     mesh, mainly managing the set of degrees of freedom. */
 class FiniteElementSpace
 {
-   friend class InterpolationGridTransfer;
-
 protected:
    /// The mesh that FE space lives on (not owned).
    Mesh *mesh;
@@ -126,11 +105,6 @@ protected:
    /// Transformation to apply to GridFunctions after space Update().
    OperatorHandle Th;
 
-   /// The element restriction operators, see GetElementRestriction().
-   mutable OperatorHandle L2E_nat, L2E_lex;
-
-   mutable Array<QuadratureInterpolator*> E2Q_array;
-
    long sequence; // should match Mesh::GetSequence
 
    void UpdateNURBS();
@@ -146,8 +120,6 @@ protected:
 
    /// Helper to get vertex, edge or face DOFs (entity=0,1,2 resp.).
    void GetEntityDofs(int entity, int index, Array<int> &dofs) const;
-   // Get degenerate face DOFs: see explanation in method implementation.
-   void GetDegenerateFaceDofs(int index, Array<int> &dofs) const;
 
    /// Calculate the cP and cR matrices for a nonconforming mesh.
    void BuildConformingInterpolation() const;
@@ -158,7 +130,6 @@ protected:
    static bool DofFinalizable(int dof, const Array<bool>& finalized,
                               const SparseMatrix& deps);
 
-   /// Replicate 'mat' in the vector dimension, according to vdim ordering mode.
    void MakeVDimMatrix(SparseMatrix &mat) const;
 
    /// GridFunction interpolation operator applicable after mesh refinement.
@@ -179,25 +150,6 @@ protected:
       virtual ~RefinementOperator();
    };
 
-   // Derefinement operator, used by the friend class InterpolationGridTransfer.
-   class DerefinementOperator : public Operator
-   {
-      const FiniteElementSpace *fine_fes; // Not owned.
-      DenseTensor localR[Geometry::NumGeom];
-      Table *coarse_elem_dof; // Owned.
-      Table coarse_to_fine;
-      Array<int> coarse_to_ref_type;
-      Array<Geometry::Type> ref_type_to_geom;
-      Array<int> ref_type_to_fine_elem_offset;
-
-   public:
-      DerefinementOperator(const FiniteElementSpace *f_fes,
-                           const FiniteElementSpace *c_fes,
-                           BilinearFormIntegrator *mass_integ);
-      virtual void Mult(const Vector &x, Vector &y) const;
-      virtual ~DerefinementOperator();
-   };
-
    // This method makes the same assumptions as the method:
    //    void GetLocalRefinementMatrices(
    //       const FiniteElementSpace &coarse_fes, Geometry::Type geom,
@@ -281,61 +233,14 @@ public:
    bool Conforming() const { return mesh->Conforming(); }
    bool Nonconforming() const { return mesh->Nonconforming(); }
 
-   /// The returned SparseMatrix is owned by the FiniteElementSpace.
    const SparseMatrix *GetConformingProlongation() const;
-
-   /// The returned SparseMatrix is owned by the FiniteElementSpace.
    const SparseMatrix *GetConformingRestriction() const;
 
-   /// The returned Operator is owned by the FiniteElementSpace.
    virtual const Operator *GetProlongationMatrix() const
    { return GetConformingProlongation(); }
-
-   /// The returned SparseMatrix is owned by the FiniteElementSpace.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { return GetConformingRestriction(); }
 
-   /// Return an Operator that converts L-vectors to E-vectors.
-   /** An L-vector is a vector of size GetVSize() which is the same size as a
-       GridFunction. An E-vector represents the element-wise discontinuous
-       version of the FE space.
-
-       The layout of the E-vector is: ND x VDIM x NE, where ND is the number of
-       degrees of freedom, VDIM is the vector dimension of the FE space, and NE
-       is the number of the mesh elements.
-
-       The parameter @a e_ordering describes how the local DOFs in each element
-       should be ordered, see ElementDofOrdering.
-
-       For discontinuous spaces, the element restriction corresponds to a
-       permutation of the degrees of freedom, implemented by the
-       L2ElementRestriction class.
-
-       The returned Operator is owned by the FiniteElementSpace. */
-   const Operator *GetElementRestriction(ElementDofOrdering e_ordering) const;
-
-   /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
-       quadrature point values and/or derivatives (Q-vectors). */
-   /** An E-vector represents the element-wise discontinuous version of the FE
-       space and can be obtained, for example, from a GridFunction using the
-       Operator returned by GetElementRestriction().
-
-       All elements will use the same IntegrationRule, @a ir as the target
-       quadrature points. */
-   const QuadratureInterpolator *GetQuadratureInterpolator(
-      const IntegrationRule &ir) const;
-
-   /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
-       quadrature point values and/or derivatives (Q-vectors). */
-   /** An E-vector represents the element-wise discontinuous version of the FE
-       space and can be obtained, for example, from a GridFunction using the
-       Operator returned by GetElementRestriction().
-
-       The target quadrature points in the elements are described by the given
-       QuadratureSpace, @a qs. */
-   const QuadratureInterpolator *GetQuadratureInterpolator(
-      const QuadratureSpace &qs) const;
-
    /// Returns vector dimension.
    inline int GetVDim() const { return vdim; }
 
@@ -668,348 +573,6 @@ public:
    void Save(std::ostream &out) const;
 };
 
-
-/** @brief Base class for transfer algorithms that construct transfer Operator%s
-    between two finite element (FE) spaces. */
-/** Generally, the two FE spaces (domain and range) can be defined on different
-    meshes. */
-class GridTransfer
-{
-protected:
-   FiniteElementSpace &dom_fes; ///< Domain FE space
-   FiniteElementSpace &ran_fes; ///< Range FE space
-
-   /** @brief Desired Operator::Type for the construction of all operators
-       defined by the underlying transfer algorithm. It can be ignored by
-       derived classes. */
-   Operator::Type oper_type;
-
-   OperatorHandle fw_t_oper; ///< Forward true-dof operator
-   OperatorHandle bw_t_oper; ///< Backward true-dof operator
-
-#ifdef MFEM_USE_MPI
-   bool parallel;
-#endif
-   bool Parallel() const
-   {
-#ifndef MFEM_USE_MPI
-      return false;
-#else
-      return parallel;
-#endif
-   }
-
-   const Operator &MakeTrueOperator(FiniteElementSpace &fes_in,
-                                    FiniteElementSpace &fes_out,
-                                    const Operator &oper,
-                                    OperatorHandle &t_oper);
-
-public:
-   /** Construct a transfer algorithm between the domain, @a dom_fes_, and
-       range, @a ran_fes_, FE spaces. */
-   GridTransfer(FiniteElementSpace &dom_fes_, FiniteElementSpace &ran_fes_);
-
-   /// Virtual destructor
-   virtual ~GridTransfer() { }
-
-   /** @brief Set the desired Operator::Type for the construction of all
-       operators defined by the underlying transfer algorithm. */
-   /** The default value is Operator::ANY_TYPE which typically corresponds to
-       a matrix-free operator representation. Note that derived classes are not
-       required to support this setting and can ignore it. */
-   void SetOperatorType(Operator::Type type) { oper_type = type; }
-
-   /** @brief Return an Operator that transfers GridFunction%s from the domain
-       FE space to GridFunction%s in the range FE space. */
-   virtual const Operator &ForwardOperator() = 0;
-
-   /** @brief Return an Operator that transfers GridFunction%s from the range
-       FE space back to GridFunction%s in the domain FE space. */
-   virtual const Operator &BackwardOperator() = 0;
-
-   /** @brief Return an Operator that transfers true-dof Vector%s from the
-       domain FE space to true-dof Vector%s in the range FE space. */
-   /** This method is implemented in the base class, based on ForwardOperator(),
-       however, derived classes can overload the construction, if necessary. */
-   virtual const Operator &TrueForwardOperator()
-   {
-      return MakeTrueOperator(dom_fes, ran_fes, ForwardOperator(), fw_t_oper);
-   }
-
-   /** @brief Return an Operator that transfers true-dof Vector%s from the
-       range FE space back to true-dof Vector%s in the domain FE space. */
-   /** This method is implemented in the base class, based on
-       BackwardOperator(), however, derived classes can overload the
-       construction, if necessary. */
-   virtual const Operator &TrueBackwardOperator()
-   {
-      return MakeTrueOperator(ran_fes, dom_fes, BackwardOperator(), bw_t_oper);
-   }
-};
-
-
-/** @brief Transfer data between a coarse mesh and an embedded refined mesh
-    using interpolation. */
-/** The forward, coarse-to-fine, transfer uses nodal interpolation. The
-    backward, fine-to-coarse, transfer is defined locally (on a coarse element)
-    as B = (F^t M_f F)^{-1} F^t M_f, where F is the forward transfer matrix, and
-    M_f is a mass matrix on the union of all fine elements comprising the coarse
-    element. Note that the backward transfer operator, B, is a left inverse of
-    the forward transfer operator, F, i.e. B F = I. Both F and B are defined in
-    reference space and do not depend on the actual physical shape of the mesh
-    elements.
-
-    It is assumed that both the coarse and the fine FiniteElementSpace%s use
-    compatible types of elements, e.g. finite elements with the same map-type
-    (VALUE, INTEGRAL, H_DIV, H_CURL - see class FiniteElement). Generally, the
-    FE spaces can have different orders, however, in order for the backward
-    operator to be well-defined, the (local) number of the fine dofs should not
-    be smaller than the number of coarse dofs. */
-class InterpolationGridTransfer : public GridTransfer
-{
-protected:
-   BilinearFormIntegrator *mass_integ; ///< Ownership depends on #own_mass_integ
-   bool own_mass_integ; ///< Ownership flag for #mass_integ
-
-   OperatorHandle F; ///< Forward, coarse-to-fine, operator
-   OperatorHandle B; ///< Backward, fine-to-coarse, operator
-
-public:
-   InterpolationGridTransfer(FiniteElementSpace &coarse_fes,
-                             FiniteElementSpace &fine_fes)
-      : GridTransfer(coarse_fes, fine_fes),
-        mass_integ(NULL), own_mass_integ(false)
-   { }
-
-   virtual ~InterpolationGridTransfer();
-
-   /** @brief Assign a mass integrator to be used in the construction of the
-       backward, fine-to-coarse, transfer operator. */
-   void SetMassIntegrator(BilinearFormIntegrator *mass_integ_,
-                          bool own_mass_integ_ = true);
-
-   virtual const Operator &ForwardOperator();
-
-   virtual const Operator &BackwardOperator();
-};
-
-
-/** @brief Transfer data between a coarse mesh and an embedded refined mesh
-    using L2 projection. */
-/** The forward, coarse-to-fine, transfer uses L2 projection. The backward,
-    fine-to-coarse, transfer is defined locally (on a coarse element) as
-    B = (F^t M_f F)^{-1} F^t M_f, where F is the forward transfer matrix, and
-    M_f is the mass matrix on the union of all fine elements comprising the
-    coarse element. Note that the backward transfer operator, B, is a left
-    inverse of the forward transfer operator, F, i.e. B F = I. Both F and B are
-    defined in physical space and, generally, vary between different mesh
-    elements.
-
-    This class currently only fully supports L2 finite element spaces and fine
-    meshes that are a uniform refinement of the coarse mesh. Generally, the
-    coarse and fine FE spaces can have different orders, however, in order for
-    the backward operator to be well-defined, the number of the fine dofs (in a
-    coarse element) should not be smaller than the number of coarse dofs.
-
-    If used on H1 finite element spaces, the transfer will be performed locally,
-    and the value of shared (interface) degrees of freedom will be determined by
-    the value of the last transfer to be performed (according to the element
-    numbering in the finite element space). As a consequence, the mass
-    conservation properties for this operator from the L2 case do not carry over
-    to H1 spaces. */
-class L2ProjectionGridTransfer : public GridTransfer
-{
-protected:
-   /** Class representing projection operator between a high-order L2 finite
-       element space on a coarse mesh, and a low-order L2 finite element space
-       on a refined mesh (LOR). We assume that the low-order space, fes_lor,
-       lives on a mesh obtained by refining the mesh of the high-order space,
-       fes_ho. */
-   class L2Projection : public Operator
-   {
-      const FiniteElementSpace &fes_ho;
-      const FiniteElementSpace &fes_lor;
-
-      int ndof_lor, ndof_ho, nref;
-
-      Table ho2lor;
-
-      DenseTensor R, P;
-
-   public:
-      L2Projection(const FiniteElementSpace &fes_ho_,
-                   const FiniteElementSpace &fes_lor_);
-      /// Perform the L2 projection onto the LOR space
-      virtual void Mult(const Vector &x, Vector &y) const;
-      /// Perform the mass conservative left-inverse prolongation operation.
-      /// This functionality is also provided as an Operator by L2Prolongation.
-      void Prolongate(const Vector &x, Vector &y) const;
-      virtual ~L2Projection() { }
-   };
-
-   /** Mass-conservative prolongation operator going in the opposite direction
-       as L2Projection. This operator is a left inverse to the L2Projection. */
-   class L2Prolongation : public Operator
-   {
-      const L2Projection &l2proj;
-
-   public:
-      L2Prolongation(const L2Projection &l2proj_) : l2proj(l2proj_) { }
-      void Mult(const Vector &x, Vector &y) const
-      {
-         l2proj.Prolongate(x, y);
-      }
-      virtual ~L2Prolongation() { }
-   };
-
-   L2Projection   *F; ///< Forward, coarse-to-fine, operator
-   L2Prolongation *B; ///< Backward, fine-to-coarse, operator
-
-public:
-   L2ProjectionGridTransfer(FiniteElementSpace &coarse_fes,
-                            FiniteElementSpace &fine_fes)
-      : GridTransfer(coarse_fes, fine_fes),
-        F(NULL), B(NULL)
-   { }
-
-   virtual const Operator &ForwardOperator();
-
-   virtual const Operator &BackwardOperator();
-};
-
-
-/// Operator that converts FiniteElementSpace L-vectors to E-vectors.
-/** Objects of this type are typically created and owned by FiniteElementSpace
-    objects, see FiniteElementSpace::GetElementRestriction(). */
-class ElementRestriction : public Operator
-{
-protected:
-   const FiniteElementSpace &fes;
-   const int ne;
-   const int vdim;
-   const bool byvdim;
-   const int ndofs;
-   const int dof;
-   const int nedofs;
-   Array<int> offsets;
-   Array<int> indices;
-
-public:
-   ElementRestriction(const FiniteElementSpace&, ElementDofOrdering);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
-/// Operator that converts L2 FiniteElementSpace L-vectors to E-vectors.
-/** Objects of this type are typically created and owned by FiniteElementSpace
-    objects, see FiniteElementSpace::GetElementRestriction(). L-vectors
-    corresponding to grid functions in L2 finite element spaces differ from
-    E-vectors only in the ordering of the degrees of freedom. */
-class L2ElementRestriction : public Operator
-{
-   const int ne;
-   const int vdim;
-   const bool byvdim;
-   const int ndof;
-public:
-   L2ElementRestriction(const FiniteElementSpace&);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
-/** @brief A class that performs interpolation from an E-vector to quadrature
-    point values and/or derivatives (Q-vectors). */
-/** An E-vector represents the element-wise discontinuous version of the FE
-    space and can be obtained, for example, from a GridFunction using the
-    Operator returned by FiniteElementSpace::GetElementRestriction().
-
-    The target quadrature points in the elements can be described either by an
-    IntegrationRule (all mesh elements must be of the same type in this case) or
-    by a QuadratureSpace. */
-class QuadratureInterpolator
-{
-protected:
-   friend class FiniteElementSpace; // Needs access to qspace and IntRule
-
-   const FiniteElementSpace *fespace;  ///< Not owned
-   const QuadratureSpace *qspace;      ///< Not owned
-   const IntegrationRule *IntRule;     ///< Not owned
-
-   mutable bool use_tensor_products;
-
-   static const int MAX_NQ2D = 100;
-   static const int MAX_ND2D = 100;
-   static const int MAX_VDIM2D = 2;
-
-   static const int MAX_NQ3D = 1000;
-   static const int MAX_ND3D = 1000;
-   static const int MAX_VDIM3D = 3;
-
-public:
-   enum EvalFlags
-   {
-      VALUES       = 1 << 0,  ///< Evaluate the values at quadrature points
-      DERIVATIVES  = 1 << 1,  ///< Evaluate the derivatives at quadrature points
-      /** @brief Assuming the derivative at quadrature points form a matrix,
-          this flag can be used to compute and store their determinants. This
-          flag can only be used in Mult(). */
-      DETERMINANTS = 1 << 2
-   };
-
-   QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const IntegrationRule &ir);
-
-   QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const QuadratureSpace &qs);
-
-   /** @brief Disable the use of tensor product evaluations, for tensor-product
-       elements, e.g. quads and hexes. */
-   /** Currently, tensor product evaluations are not implemented and this method
-       has no effect. */
-   void DisableTensorProducts(bool disable = true) const
-   { use_tensor_products = !disable; }
-
-   /// Interpolate the E-vector @a e_vec to quadrature points.
-   /** The @a eval_flags are a bitwise mask of constants from the EvalFlags
-       enumeration. When the VALUES flag is set, the values at quadrature points
-       are computed and stored in the Vector @a q_val. Similarly, when the flag
-       DERIVATIVES is set, the derivatives are computed and stored in @a q_der.
-       When the DETERMINANTS flags is set, it is assumed that the derivatives
-       form a matrix at each quadrature point (i.e. the associated
-       FiniteElementSpace is a vector space) and their determinants are computed
-       and stored in @a q_det. */
-   void Mult(const Vector &e_vec, unsigned eval_flags,
-             Vector &q_val, Vector &q_der, Vector &q_det) const;
-
-   /// Perform the transpose operation of Mult(). (TODO)
-   void MultTranspose(unsigned eval_flags, const Vector &q_val,
-                      const Vector &q_der, Vector &e_vec) const;
-
-   // Compute kernels follow (cannot be private or protected with nvcc)
-
-   /// Template compute kernel for 2D.
-   template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Eval2D(const int NE,
-                      const int vdim,
-                      const DofToQuad &maps,
-                      const Vector &e_vec,
-                      Vector &q_val,
-                      Vector &q_der,
-                      Vector &q_det,
-                      const int eval_flags);
-
-   /// Template compute kernel for 3D.
-   template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Eval3D(const int NE,
-                      const int vdim,
-                      const DofToQuad &maps,
-                      const Vector &e_vec,
-                      Vector &q_val,
-                      Vector &q_der,
-                      Vector &q_det,
-                      const int eval_flags);
-};
-
 }
 
 #endif

fem/gridfunc.cpp

@@ -30,9 +30,6 @@ using namespace std;
 GridFunction::GridFunction(Mesh *m, std::istream &input)
    : Vector()
 {
-   // Grid functions are stored on the device
-   UseDevice(true);
-
    fes = new FiniteElementSpace;
    fec = fes->Load(m, input);
 
@@ -63,8 +60,6 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
 
 GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
 {
-   UseDevice(true);
-
    // all GridFunctions must have the same FE collection, vdim, ordering
    int vdim, ordering;
 
@@ -168,7 +163,6 @@ void GridFunction::Update()
       Vector old_data;
       old_data.Swap(*this);
       SetSize(T->Height());
-      UseDevice(true);
       T->Mult(old_data, *this);
    }
    else
@@ -198,9 +192,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
    MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
    if (f != fes) { Destroy(); }
    fes = f;
-   v.UseDevice(true);
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
-                    fes->GetVSize(), true);
+   NewDataAndSize((double *)v + v_offset, fes->GetVSize());
    sequence = fes->GetSequence();
 }
 
@@ -223,16 +215,13 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
    if (!f->GetProlongationMatrix())
    {
       MakeRef(f, tv, tv_offset);
-      t_vec.NewMemoryAndSize(data, size, false);
+      t_vec.NewDataAndSize(data, size);
    }
    else
    {
       MFEM_ASSERT(tv.Size() >= tv_offset + f->GetTrueVSize(), "");
       SetSpace(f); // works in parallel
-      tv.UseDevice(true);
-      const int tv_size = f->GetTrueVSize();
-      t_vec.NewMemoryAndSize(Memory<double>(tv.GetMemory(), tv_offset, tv_size),
-                             tv_size, true);
+      t_vec.NewDataAndSize(&tv(tv_offset), f->GetTrueVSize());
    }
 }
 
@@ -313,7 +302,7 @@ int GridFunction::VectorDim() const
    {
       fe = fes->GetFE(0);
    }
-   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
+   if (fe->GetRangeType() == FiniteElement::SCALAR)
    {
       return fes->GetVDim();
    }
@@ -326,7 +315,7 @@ void GridFunction::GetTrueDofs(Vector &tv) const
    if (!R)
    {
       // R is identity -> make tv a reference to *this
-      tv.NewDataAndSize(const_cast<double*>((const double*)data), size);
+      tv.NewDataAndSize(data, size);
    }
    else
    {
@@ -1378,7 +1367,7 @@ void GridFunction::AccumulateAndCountBdrValues(
          if (vdofs.Size() == 0) { continue; }
 
          transf = mesh->GetEdgeTransformation(edge);
-         transf->Attribute = -1; // TODO: set the boundary attribute
+         transf->Attribute = -1; // FIXME: set the boundary attribute
          fe = fes->GetEdgeElement(edge);
          if (!vcoeff)
          {
@@ -1482,7 +1471,7 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
          if (dofs.Size() == 0) { continue; }
 
          T = mesh->GetEdgeTransformation(edge);
-         T->Attribute = -1; // TODO: set the boundary attribute
+         T->Attribute = -1; // FIXME: set the boundary attribute
          fe = fes->GetEdgeElement(edge);
          lvec.SetSize(fe->GetDof());
          fe->Project(vcoeff, *T, lvec);
@@ -1716,7 +1705,6 @@ void GridFunction::ProjectDiscCoefficient(VectorCoefficient &coeff,
    Array<int> vdofs;
    Vector vals;
 
-   HostWrite();
    // maximal element attribute for each dof
    dof_attr.SetSize(fes->GetVSize());
    dof_attr = -1;
@@ -1788,7 +1776,6 @@ void GridFunction::ProjectBdrCoefficient(VectorCoefficient &vcoeff,
 void GridFunction::ProjectBdrCoefficient(Coefficient *coeff[], Array<int> &attr)
 {
    Array<int> values_counter;
-   this->HostReadWrite();
    AccumulateAndCountBdrValues(coeff, NULL, attr, values_counter);
    ComputeMeans(ARITHMETIC, values_counter);
 #ifdef MFEM_DEBUG

fem/gridfunc.hpp

@@ -68,16 +68,15 @@ protected:
 
 public:
 
-   GridFunction() { fes = NULL; fec = NULL; sequence = 0; UseDevice(true); }
+   GridFunction() { fes = NULL; fec = NULL; sequence = 0; }
 
    /// Copy constructor. The internal true-dof vector #t_vec is not copied.
    GridFunction(const GridFunction &orig)
-      : Vector(orig), fes(orig.fes), fec(NULL), sequence(orig.sequence)
-   { UseDevice(true); }
+      : Vector(orig), fes(orig.fes), fec(NULL), sequence(orig.sequence) { }
 
    /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
    GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; fec = NULL; sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec = NULL; sequence = f->GetSequence(); }
 
    /// Construct a GridFunction using previously allocated array @a data.
    /** The GridFunction does not assume ownership of @a data which is assumed to
@@ -85,9 +84,8 @@ public:
        for externally allocated array, the pointer @a data can be NULL. The data
        array can be replaced later using the method SetData().
     */
-   GridFunction(FiniteElementSpace *f, double *data)
-      : Vector(data, f->GetVSize())
-   { fes = f; fec = NULL; sequence = f->GetSequence(); UseDevice(true); }
+   GridFunction(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
+   { fes = f; fec = NULL; sequence = f->GetSequence(); }
 
    /// Construct a GridFunction on the given Mesh, using the data from @a input.
    /** The content of @a input should be in the format created by the method
@@ -106,9 +104,7 @@ public:
    GridFunction &operator=(const GridFunction &rhs)
    { return operator=((const Vector &)rhs); }
 
-   /// Make the GridFunction the owner of #fec and #fes.
-   /** If the new FiniteElementCollection, @a _fec, is NULL, ownership of #fec
-       and #fes is taken away. */
+   /// Make the GridFunction the owner of 'fec' and 'fes'
    void MakeOwner(FiniteElementCollection *_fec) { fec = _fec; }
 
    FiniteElementCollection *OwnFEC() { return fec; }
@@ -126,7 +122,6 @@ public:
 
    /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
    /// then `tv` will be set to point to the data of `*this`.
-   /** @warning This method breaks const-ness when all dofs are true. */
    void GetTrueDofs(Vector &tv) const;
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.
@@ -434,8 +429,6 @@ public:
    /** The GridFunction is resized using the SetSize() method. */
    virtual void SetSpace(FiniteElementSpace *f);
 
-   using Vector::MakeRef;
-
    /** @brief Make the GridFunction reference external data on a new
        FiniteElementSpace. */
    /** This method changes the FiniteElementSpace associated with the
@@ -707,7 +700,7 @@ inline void QuadratureFunction::GetElementValues(int idx, Vector &values) const
    const int s_offset = qspace->element_offsets[idx];
    const int sl_size = qspace->element_offsets[idx+1] - s_offset;
    values.SetSize(vdim*sl_size);
-   const double *q = data + vdim*s_offset;
+   double *q = data + vdim*s_offset;
    for (int i = 0; i<values.Size(); i++)
    {
       values(i) = *(q++);
@@ -727,14 +720,12 @@ inline void QuadratureFunction::GetElementValues(int idx,
    const int s_offset = qspace->element_offsets[idx];
    const int sl_size = qspace->element_offsets[idx+1] - s_offset;
    values.SetSize(vdim, sl_size);
-   const double *q = data + vdim*s_offset;
+   double *q = data + vdim*s_offset;
    for (int j = 0; j<sl_size; j++)
-   {
       for (int i = 0; i<vdim; i++)
       {
          values(i,j) = *(q++);
       }
-   }
 }
 
 } // namespace mfem

fem/intrules.cpp

@@ -78,19 +78,6 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
    }
 }
 
-const Array<double> &IntegrationRule::GetWeights() const
-{
-   if (weights.Size() != GetNPoints())
-   {
-      weights.SetSize(GetNPoints());
-      for (int i = 0; i < GetNPoints(); i++)
-      {
-         weights[i] = IntPoint(i).weight;
-      }
-   }
-   return weights;
-}
-
 void IntegrationRule::GrundmannMollerSimplexRule(int s, int n)
 {
    // for pow on older compilers
@@ -899,7 +886,7 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
 
    if (!HaveIntRule(*ir_array, Order))
    {
-#ifdef MFEM_USE_LEGACY_OPENMP
+#ifdef MFEM_USE_OPENMP
       #pragma omp critical
 #endif
       {
@@ -1608,20 +1595,20 @@ IntegrationRule *IntegrationRules::TetrahedronIntegrationRule(int Order)
 // Integration rules for reference prism
 IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)
 {
-   const IntegrationRule & irt = Get(Geometry::TRIANGLE, Order);
-   const IntegrationRule & irs = Get(Geometry::SEGMENT, Order);
-   int nt = irt.GetNPoints();
-   int ns = irs.GetNPoints();
+   IntegrationRule * irt = GenerateIntegrationRule(Geometry::TRIANGLE, Order);
+   IntegrationRule * irs = GenerateIntegrationRule(Geometry::SEGMENT, Order);
+   int nt = irt->GetNPoints();
+   int ns = irs->GetNPoints();
    AllocIntRule(PrismIntRules, Order);
    PrismIntRules[Order] = new IntegrationRule(nt * ns);
 
    for (int ks=0; ks<ns; ks++)
    {
-      const IntegrationPoint & ips = irs.IntPoint(ks);
+      const IntegrationPoint & ips = irs->IntPoint(ks);
       for (int kt=0; kt<nt; kt++)
       {
          int kp = ks * nt + kt;
-         const IntegrationPoint & ipt = irt.IntPoint(kt);
+         const IntegrationPoint & ipt = irt->IntPoint(kt);
          IntegrationPoint & ipp = PrismIntRules[Order]->IntPoint(kp);
          ipp.x = ipt.x;
          ipp.y = ipt.y;

fem/intrules.hpp

@@ -87,9 +87,6 @@ class IntegrationRule : public Array<IntegrationPoint>
 private:
    friend class IntegrationRules;
    int Order;
-   /** @brief The quadrature weights gathered as a contiguous array. Created
-       by request with the method GetWeights(). */
-   mutable Array<double> weights;
 
    /// Define n-simplex rule (triangle/tetrahedron for n=2/3) of order (2s+1)
    void GrundmannMollerSimplexRule(int s, int n = 3);
@@ -242,11 +239,6 @@ public:
    /// Returns a const reference to the i-th integration point
    const IntegrationPoint &IntPoint(int i) const { return (*this)[i]; }
 
-   /// Return the quadrature weights in a contiguous array.
-   /** If a contiguous array is not required, the weights can be accessed with
-       a call like this: `IntPoint(i).weight`. */
-   const Array<double> &GetWeights() const;
-
    /// Destroys an IntegrationRule object
    ~IntegrationRule() { }
 };
@@ -333,9 +325,6 @@ private:
       return Order | 1; // valid for all quad_type's
    }
 
-   /// The following methods allocate new IntegrationRule objects without
-   /// checking if they already exist.  To avoid memory leaks use
-   /// IntegrationRules::Get(int GeomType, int Order) instead.
    IntegrationRule *GenerateIntegrationRule(int GeomType, int Order);
    IntegrationRule *PointIntegrationRule(int Order);
    IntegrationRule *SegmentIntegrationRule(int Order);

fem/linearform.cpp

@@ -19,9 +19,6 @@ namespace mfem
 LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
    : Vector(f->GetVSize())
 {
-   // Linear forms are stored on the device
-   UseDevice(true);
-
    fes = f;
    extern_lfs = 1;
 
@@ -86,10 +83,6 @@ void LinearForm::Assemble()
 
    Vector::operator=(0.0);
 
-   // The above operation is executed on device because of UseDevice().
-   // The first use of AddElementVector() below will move it back to host
-   // because both 'vdofs' and 'elemvect' are on host.
-
    if (dlfi.Size())
    {
       for (i = 0; i < fes -> GetNE(); i++)
@@ -138,11 +131,7 @@ void LinearForm::Assemble()
          eltrans = fes -> GetBdrElementTransformation (i);
          for (int k=0; k < blfi.Size(); k++)
          {
-            if (blfi_marker[k] &&
-                (*blfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
             blfi[k]->AssembleRHSElementVect(*fes->GetBE(i), *eltrans, elemvect);
-
             AddElementVector (vdofs, elemvect);
          }
       }

fem/linearform.hpp

@@ -64,7 +64,7 @@ public:
    /// Creates linear form associated with FE space @a *f.
    /** The pointer @a f is not owned by the newly constructed object. */
    LinearForm(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; extern_lfs = 0; UseDevice(true); }
+   { fes = f; extern_lfs = 0; }
 
    /** @brief Create a LinearForm on the FiniteElementSpace @a f, using the
        same integrators as the LinearForm @a lf.
@@ -79,7 +79,7 @@ public:
    /** The associated FiniteElementSpace can be set later using one of the
        methods: Update(FiniteElementSpace *) or
        Update(FiniteElementSpace *, Vector &, int). */
-   LinearForm() { fes = NULL; extern_lfs = 0; UseDevice(true); }
+   LinearForm() { fes = NULL; extern_lfs = 0; }
 
    /// Copy assignment. Only the data of the base class Vector is copied.
    /** It is assumed that this object and @a rhs use FiniteElementSpace%s that

fem/lininteg.cpp

@@ -181,7 +181,7 @@ void VectorDomainLFIntegrator::AssembleRHSElementVect(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int intorder = 2*el.GetOrder();
+      int intorder = el.GetOrder() + 1;
       ir = &IntRules.Get(el.GetGeomType(), intorder);
    }
 
@@ -240,7 +240,7 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int intorder = 2*el.GetOrder();
+      int intorder = el.GetOrder() + 1;
       ir = &IntRules.Get(el.GetGeomType(), intorder);
    }
 
@@ -275,7 +275,7 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int intorder = 2*el.GetOrder();
+      int intorder = el.GetOrder() + 1;
       ir = &IntRules.Get(Tr.FaceGeom, intorder);
    }
 
@@ -350,6 +350,7 @@ void VectorFEDomainLFIntegrator::AssembleDeltaElementVect(
    vshape.Mult(vec, elvect);
 }
 
+
 void VectorBoundaryFluxLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -396,26 +397,19 @@ void VectorFEBoundaryFluxLFIntegrator::AssembleRHSElementVect(
    if (ir == NULL)
    {
       int intorder = 2*el.GetOrder();  // <----------
-      if (F == NULL)
-      {
-         intorder -= el.GetOrder() + 1;
-      }
       ir = &IntRules.Get(el.GetGeomType(), intorder);
    }
 
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetIntPoint (&ip);
+      double val = ip.weight*F.Eval(Tr, ip);
+
       el.CalcShape(ip, shape);
 
-      double val = ip.weight;
-      if (F)
-      {
-         Tr.SetIntPoint (&ip);
-         val *= F->Eval(Tr, ip);
-      }
-
-      elvect.Add(val, shape);
+      add(elvect, val, shape, elvect);
    }
 }
 

fem/lininteg.hpp

@@ -279,12 +279,11 @@ public:
 class VectorFEBoundaryFluxLFIntegrator : public LinearFormIntegrator
 {
 private:
-   Coefficient *F;
+   Coefficient &F;
    Vector shape;
 
 public:
-   VectorFEBoundaryFluxLFIntegrator() : F(NULL) { }
-   VectorFEBoundaryFluxLFIntegrator(Coefficient &f) : F(&f) { }
+   VectorFEBoundaryFluxLFIntegrator(Coefficient &f) : F(f) { }
 
    virtual void AssembleRHSElementVect(const FiniteElement &el,
                                        ElementTransformation &Tr,

fem/nonlinearform.cpp

@@ -65,8 +65,6 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
    Vector el_x;
    const FiniteElement *fe;
    ElementTransformation *T;
-   Mesh *mesh = fes->GetMesh();
-
    double energy = 0.0;
 
    if (dnfi.Size())
@@ -86,81 +84,14 @@ double NonlinearForm::GetGridFunctionEnergy(const Vector &x) const
 
    if (fnfi.Size())
    {
-      FaceElementTransformations *tr;
-      const FiniteElement *fe1, *fe2;
-      Array<int> vdofs2;
-
-      for (int i = 0; i < mesh->GetNumFaces(); i++)
-      {
-         tr = mesh->GetInteriorFaceTransformations(i);
-         if (tr != NULL)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            fes->GetElementVDofs(tr->Elem2No, vdofs2);
-            vdofs.Append (vdofs2);
-            x.GetSubVector(vdofs, el_x);
-            fe1 = fes->GetFE(tr->Elem1No);
-            fe2 = fes->GetFE(tr->Elem2No);
-            for (int k = 0; k < fnfi.Size(); k++)
-            {
-               energy += fnfi[k]->GetFaceEnergy(*fe1, *fe2, *tr, el_x);
-            }
-         }
-      }
+      MFEM_ABORT("TODO: add energy contribution from interior face terms");
    }
 
    if (bfnfi.Size())
    {
-      FaceElementTransformations *tr;
-      const FiniteElement *fe1, *fe2;
-
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < bfnfi.Size(); k++)
-      {
-         if (bfnfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *bfnfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary face integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         tr = mesh->GetBdrFaceTransformations (i);
-         if (tr != NULL)
-         {
-            fes->GetElementVDofs(tr->Elem1No, vdofs);
-            x.GetSubVector(vdofs, el_x);
-
-            fe1 = fes->GetFE(tr->Elem1No);
-            // The fe2 object is really a dummy and not used on the boundaries,
-            // but we can't dereference a NULL pointer, and we don't want to
-            // actually make a fake element.
-            fe2 = fe1;
-            for (int k = 0; k < bfnfi.Size(); k++)
-            {
-               if (bfnfi_marker[k] &&
-                   (*bfnfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-               energy += bfnfi[k]->GetFaceEnergy(*fe1, *fe2, *tr, el_x);
-            }
-         }
-      }
+      MFEM_ABORT("TODO: add energy contribution from boundary face terms");
    }
+
    return energy;
 }
 

fem/nonlinearform.hpp

@@ -111,7 +111,7 @@ public:
        be fes->GetVSize(). */
    double GetGridFunctionEnergy(const Vector &x) const;
 
-   /// Compute the energy corresponding to the state @a x.
+   /// Compute the enery corresponding to the state @a x.
    /** In general, @a x may have non-homogeneous essential boundary values.
 
        The state @a x must be a true-dof vector. */

fem/nonlininteg.cpp

@@ -55,14 +55,6 @@ double NonlinearFormIntegrator::GetElementEnergy(
    return 0.0;
 }
 
-double NonlinearFormIntegrator::GetFaceEnergy(
-   const FiniteElement &el1, const FiniteElement &el2,
-   FaceElementTransformations &Tr, const Vector &elfun)
-{
-   mfem_error("NonlinearFormIntegrator::GetFaceEnergy"
-              " is not overloaded!");
-   return 0.0;
-}
 
 void BlockNonlinearFormIntegrator::AssembleElementVector(
    const Array<const FiniteElement *> &el,

fem/nonlininteg.hpp

@@ -63,17 +63,11 @@ public:
                                  FaceElementTransformations &Tr,
                                  const Vector &elfun, DenseMatrix &elmat);
 
-   /// Compute the local energy/functional
+   /// Compute the local energy
    virtual double GetElementEnergy(const FiniteElement &el,
                                    ElementTransformation &Tr,
                                    const Vector &elfun);
 
-   /// Compute the face(s) contribution to the energy/functional
-   virtual double GetFaceEnergy(const FiniteElement &el1,
-                                const FiniteElement &el2,
-                                FaceElementTransformations &Tr,
-                                const Vector &elfun);
-
    virtual ~NonlinearFormIntegrator() { }
 };
 

fem/pbilinearform.cpp

@@ -203,14 +203,7 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
       vdofs1.Copy(vdofs_all);
       for (int j = 0; j < vdofs2.Size(); j++)
       {
-         if (vdofs2[j] >= 0)
-         {
-            vdofs2[j] += height;
-         }
-         else
-         {
-            vdofs2[j] -= height;
-         }
+         vdofs2[j] += height;
       }
       vdofs_all.Append(vdofs2);
       for (int k = 0; k < fbfi.Size(); k++)
@@ -291,12 +284,6 @@ void ParBilinearForm::FormLinearSystem(
    const Array<int> &ess_tdof_list, Vector &x, Vector &b,
    OperatorHandle &A, Vector &X, Vector &B, int copy_interior)
 {
-   if (ext)
-   {
-      ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-      return;
-   }
-
    // Finish the matrix assembly and perform BC elimination, storing the
    // eliminated part of the matrix.
    FormSystemMatrix(ess_tdof_list, A);
@@ -340,12 +327,6 @@ void ParBilinearForm::FormLinearSystem(
 void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
                                        OperatorHandle &A)
 {
-   if (ext)
-   {
-      ext->FormSystemMatrix(ess_tdof_list, A);
-      return;
-   }
-
    // Finish the matrix assembly and perform BC elimination, storing the
    // eliminated part of the matrix.
    if (static_cond)
@@ -388,12 +369,6 @@ void ParBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 void ParBilinearForm::RecoverFEMSolution(
    const Vector &X, const Vector &b, Vector &x)
 {
-   if (ext)
-   {
-      ext->RecoverFEMSolution(X, b, x);
-      return;
-   }
-
    const Operator &P = *pfes->GetProlongationMatrix();
 
    if (static_cond)
@@ -492,8 +467,8 @@ void ParMixedBilinearForm::TrueAddMult(const Vector &x, Vector &y,
 
 HypreParMatrix* ParDiscreteLinearOperator::ParallelAssemble() const
 {
-   MFEM_ASSERT(mat, "Matrix is not assembled");
-   MFEM_ASSERT(mat->Finalized(), "Matrix is not finalized");
+   MFEM_ASSERT(mat, "matrix is not assembled");
+   MFEM_ASSERT(mat->Finalized(), "matrix is not finalized");
    SparseMatrix* RA = mfem::Mult(*range_fes->GetRestrictionMatrix(), *mat);
    HypreParMatrix* P = domain_fes->Dof_TrueDof_Matrix();
    HypreParMatrix* RAP = P->LeftDiagMult(*RA, range_fes->GetTrueDofOffsets());
@@ -504,7 +479,7 @@ HypreParMatrix* ParDiscreteLinearOperator::ParallelAssemble() const
 void ParDiscreteLinearOperator::GetParBlocks(Array2D<HypreParMatrix *> &blocks)
 const
 {
-   MFEM_VERIFY(mat->Finalized(), "Local matrix needs to be finalized for "
+   MFEM_VERIFY(mat->Finalized(), "local matrix needs to be finalized for "
                "GetParBlocks");
 
    HypreParMatrix* RLP = ParallelAssemble();

fem/pbilinearform.hpp

@@ -75,8 +75,7 @@ public:
        those rows. Must be called before the first Assemble call. */
    void KeepNbrBlock(bool knb = true) { keep_nbr_block = knb; }
 
-   /** @brief Set the operator type id for the parallel matrix/operator when
-       using AssemblyLevel::FULL. */
+   /// Set the operator type id for the parallel matrix/operator.
    /** If using static condensation or hybridization, call this method *after*
        enabling it. */
    void SetOperatorType(Operator::Type tid)
@@ -164,15 +163,65 @@ public:
    virtual const Operator *GetRestriction() const
    { return pfes->GetRestrictionMatrix(); }
 
-   using BilinearForm::FormLinearSystem;
-   using BilinearForm::FormSystemMatrix;
+   /** Form the linear system A X = B, corresponding to the current bilinear
+       form and b(.), by applying any necessary transformations such as:
+       eliminating boundary conditions; applying conforming constraints for
+       non-conforming AMR; parallel assembly; static condensation;
+       hybridization.
 
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
-                                 Vector &b, OperatorHandle &A, Vector &X,
-                                 Vector &B, int copy_interior = 0);
+       The ParGridFunction-size vector x must contain the essential b.c. The
+       ParBilinearForm and the ParLinearForm-size vector b must be assembled.
 
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A);
+       The vector X is initialized with a suitable initial guess: when using
+       hybridization, the vector X is set to zero; otherwise, the essential
+       entries of X are set to the corresponding b.c. and all other entries are
+       set to zero (copy_interior == 0) or copied from x (copy_interior != 0).
+
+       This method can be called multiple times (with the same ess_tdof_list
+       array) to initialize different right-hand sides and boundary condition
+       values.
+
+       After solving the linear system, the finite element solution x can be
+       recovered by calling RecoverFEMSolution (with the same vectors X, b, and
+       x). */
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0);
+
+   /** Version of the method FormLinearSystem() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         OpType &A, Vector &X, Vector &B,
+                         int copy_interior = 0)
+   {
+      OperatorHandle Ah;
+      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   /// Form the linear system matrix @a A, see FormLinearSystem() for details.
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+
+   /** Version of the method FormSystemMatrix() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormSystemMatrix(ess_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
 
    /** Call this method after solving a linear system constructed using the
        FormLinearSystem method to recover the solution as a ParGridFunction-size

fem/pfespace.cpp

@@ -14,7 +14,6 @@
 #ifdef MFEM_USE_MPI
 
 #include "pfespace.hpp"
-#include "../general/forall.hpp"
 #include "../general/sort_pairs.hpp"
 #include "../mesh/mesh_headers.hpp"
 #include "../general/binaryio.hpp"
@@ -98,8 +97,6 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
 
    gcomm = NULL;
 
-   gfdofs = NULL;
-
    P = NULL;
    Pconf = NULL;
    R = NULL;
@@ -150,37 +147,20 @@ void ParFiniteElementSpace::Construct()
       // cut space.
       ConstructTrueDofs();
 
-      ngedofs = ngfdofs = 0;
-      gfdofs = NULL;
-
       // calculate number of ghost DOFs
       ngvdofs = pncmesh->GetNGhostVertices()
                 * fec->DofForGeometry(Geometry::POINT);
 
+      ngedofs = ngfdofs = 0;
       if (pmesh->Dimension() > 1)
       {
          ngedofs = pncmesh->GetNGhostEdges()
                    * fec->DofForGeometry(Geometry::SEGMENT);
       }
-
       if (pmesh->Dimension() > 2)
       {
-         if (fdofs != NULL) // have mixed faces
-         {
-            gfdofs = new int[pncmesh->GetNGhostFaces()+1];
-            gfdofs[0] = 0;
-            for (int i = 0; i < pncmesh->GetNGhostFaces(); i++)
-            {
-               int ghost = pncmesh->GetNFaces() + i;
-               ngfdofs += fec->DofForGeometry(pncmesh->GetFaceGeometry(ghost));
-               gfdofs[i+1] = ngfdofs;
-            }
-         }
-         else
-         {
-            ngfdofs = pncmesh->GetNGhostFaces()
-                      * fec->DofForGeometry(pncmesh->GetFaceGeometry(0));
-         }
+         ngfdofs = pncmesh->GetNGhostFaces()
+                   * fec->DofForGeometry(pncmesh->GetGhostFaceGeometry(0));
       }
 
       // total number of ghost DOFs. Ghost DOFs start at index 'ndofs', i.e.,
@@ -767,14 +747,12 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
    // Verify that in boolean arithmetic: P^T ess_dofs = R ess_dofs.
    Array<int> true_ess_dofs2(true_ess_dofs.Size());
    HypreParMatrix *Pt = Dof_TrueDof_Matrix()->Transpose();
-   const int *ess_dofs_data = ess_dofs.HostRead();
-   Pt->BooleanMult(1, ess_dofs_data, 0, true_ess_dofs2);
+   Pt->BooleanMult(1, ess_dofs, 0, true_ess_dofs2);
    delete Pt;
    int counter = 0;
-   const int *ted = true_ess_dofs.HostRead();
    for (int i = 0; i < true_ess_dofs.Size(); i++)
    {
-      if (bool(ted[i]) != bool(true_ess_dofs2[i])) { counter++; }
+      if (bool(true_ess_dofs[i]) != bool(true_ess_dofs2[i])) { counter++; }
    }
    MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter);
 #endif
@@ -876,20 +854,7 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
 {
    if (Conforming())
    {
-      if (!Pconf)
-      {
-         if (!Device::Allows(Backend::DEVICE_MASK))
-         {
-            Pconf = new ConformingProlongationOperator(*this);
-         }
-         else
-         {
-            if (NRanks > 1)
-            {
-               Pconf = new DeviceConformingProlongationOperator(*this);
-            }
-         }
-      }
+      if (!Pconf) { Pconf = new ConformingProlongationOperator(*this); }
       return Pconf;
    }
    else
@@ -937,15 +902,11 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
       {
          GetElementVDofs(my_elems[i], ldofs);
          for (int j = 0; j < ldofs.Size(); j++)
-         {
-            int ldof = (ldofs[j] >= 0 ? ldofs[j] : -1-ldofs[j]);
-
-            if (ldof_marker[ldof] != fn)
+            if (ldof_marker[ldofs[j]] != fn)
             {
-               ldof_marker[ldof] = fn;
+               ldof_marker[ldofs[j]] = fn;
                send_face_nbr_ldof.AddAColumnInRow(fn);
             }
-         }
          send_nbr_elem_dof.AddColumnsInRow(send_el_off[fn] + i, ldofs.Size());
       }
 
@@ -999,11 +960,9 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
          GetElementVDofs(my_elems[i], ldofs);
          for (int j = 0; j < ldofs.Size(); j++)
          {
-            int ldof = (ldofs[j] >= 0 ? ldofs[j] : -1-ldofs[j]);
-
-            if (ldof_marker[ldof] != fn)
+            if (ldof_marker[ldofs[j]] != fn)
             {
-               ldof_marker[ldof] = fn;
+               ldof_marker[ldofs[j]] = fn;
                send_face_nbr_ldof.AddConnection(fn, ldofs[j]);
             }
          }
@@ -1024,14 +983,12 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
 
       for (int i = 0; i < num_ldofs; i++)
       {
-         int ldof = (ldofs[i] >= 0 ? ldofs[i] : -1-ldofs[i]);
-         ldof_marker[ldof] = i;
+         ldof_marker[ldofs[i]] = i;
       }
 
       for ( ; j < j_end; j++)
       {
-         int ldof = (send_J[j] >= 0 ? send_J[j] : -1-send_J[j]);
-         send_J[j] = (send_J[j] >= 0 ? ldof_marker[ldof] : -1-ldof_marker[ldof]);
+         send_J[j] = ldof_marker[send_J[j]];
       }
    }
 
@@ -1066,14 +1023,7 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
 
       for ( ; j < j_end; j++)
       {
-         if (recv_J[j] >= 0)
-         {
-            recv_J[j] += shift;
-         }
-         else
-         {
-            recv_J[j] -= shift;
-         }
+         recv_J[j] += shift;
       }
    }
 
@@ -1122,15 +1072,8 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
    for (int fn = 0, j = 0; fn < num_face_nbrs; fn++)
    {
       for (int j_end = face_nbr_ldof.GetI()[fn+1]; j < j_end; j++)
-      {
-         int ldof = face_nbr_ldof.GetJ()[j];
-         if (ldof < 0)
-         {
-            ldof = -1-ldof;
-         }
-
-         face_nbr_glob_dof_map[j] = dof_face_nbr_offsets[fn] + ldof;
-      }
+         face_nbr_glob_dof_map[j] =
+            dof_face_nbr_offsets[fn] + face_nbr_ldof.GetJ()[j];
    }
 
    MPI_Waitall(num_face_nbrs, send_requests, statuses);
@@ -1343,18 +1286,20 @@ void ParFiniteElementSpace::GetGhostEdgeDofs(const MeshId &edge_id,
 void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
                                              Array<int> &dofs) const
 {
-   int nfv, V[4], E[4], Eo[4];
-   nfv = pmesh->pncmesh->GetFaceVerticesEdges(face_id, V, E, Eo);
+   const int ghost_face_index = face_id.index - pncmesh->GetNFaces();
+   MFEM_ASSERT(pncmesh->GetGhostFaceGeometry(ghost_face_index)
+               == Geometry::SQUARE, "");
 
    int nv = fec->DofForGeometry(Geometry::POINT);
    int ne = fec->DofForGeometry(Geometry::SEGMENT);
-   int nf = fec->DofForGeometry((nfv == 3) ?
-                                Geometry::TRIANGLE : Geometry::SQUARE);
+   int nf = fec->DofForGeometry(Geometry::SQUARE);
+   dofs.SetSize(4*nv + 4*ne + nf);
 
-   dofs.SetSize(nfv*(nv + ne) + nf);
+   int V[4], E[4], Eo[4];
+   pmesh->pncmesh->GetFaceVerticesEdges(face_id, V, E, Eo);
 
    int offset = 0;
-   for (int i = 0; i < nfv; i++)
+   for (int i = 0; i < 4; i++)
    {
       int ghost = pncmesh->GetNVertices();
       int first = (V[i] < ghost) ? V[i]*nv : (ndofs + (V[i] - ghost)*nv);
@@ -1364,7 +1309,7 @@ void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
       }
    }
 
-   for (int i = 0; i < nfv; i++)
+   for (int i = 0; i < 4; i++)
    {
       int ghost = pncmesh->GetNEdges();
       int first = (E[i] < ghost) ? nvdofs + E[i]*ne
@@ -1377,10 +1322,8 @@ void ParFiniteElementSpace::GetGhostFaceDofs(const MeshId &face_id,
       }
    }
 
-   const int ghost_face_index = face_id.index - pncmesh->GetNFaces();
-   int first = ndofs + ngvdofs + ngedofs;
-   first += gfdofs ? gfdofs[ghost_face_index] : nf*ghost_face_index;
-
+   // Assuming all ghost faces have the same number of dofs:
+   int first = ndofs + ngvdofs + ngedofs + ghost_face_index*nf;
    for (int j = 0; j < nf; j++)
    {
       dofs[offset++] = first + j;
@@ -1422,19 +1365,12 @@ void ParFiniteElementSpace::GetBareDofs(int entity, int index,
          break;
 
       default:
-         ned = fec->DofForGeometry(pncmesh->GetFaceGeometry(index));
+         MFEM_ASSERT(!pmesh->HasGeometry(Geometry::TRIANGLE), "");
+         ned = fec->DofForGeometry(Geometry::SQUARE);
          ghost = pncmesh->GetNFaces();
-
-         if (index < ghost) // regular face
-         {
-            first = nvdofs + nedofs + (fdofs ? fdofs[index] : index*ned);
-         }
-         else // ghost face
-         {
-            index -= ghost;
-            first = ndofs + ngvdofs + ngedofs +
-                    (gfdofs ? gfdofs[index] : index*ned);
-         }
+         first = (index < ghost)
+                 ? nvdofs + nedofs + index*ned // regular face
+                 : ndofs + ngvdofs + ngedofs + (index - ghost)*ned; // ghost
          break;
    }
 
@@ -1470,30 +1406,16 @@ int ParFiniteElementSpace::PackDof(int entity, int index, int edof) const
                 : ndofs + ngvdofs + (index - ghost)*ned + edof; // ghost edge
 
       default:
+         MFEM_ASSERT(!pmesh->HasGeometry(Geometry::TRIANGLE), "");
          ghost = pncmesh->GetNFaces();
-         ned = fec->DofForGeometry(pncmesh->GetFaceGeometry(index));
+         ned = fec->DofForGeometry(Geometry::SQUARE);
 
-         if (index < ghost) // regular face
-         {
-            return nvdofs + nedofs + (fdofs ? fdofs[index] : index*ned) + edof;
-         }
-         else // ghost face
-         {
-            index -= ghost;
-            return ndofs + ngvdofs + ngedofs +
-                   (gfdofs ? gfdofs[index] : index*ned) + edof;
-         }
+         return (index < ghost)
+                ? nvdofs + nedofs + index*ned + edof // regular face
+                : ndofs + ngvdofs + ngedofs + (index - ghost)*ned + edof; //ghost
    }
 }
 
-static int bisect(int* array, int size, int value)
-{
-   int* end = array + size;
-   int* pos = std::upper_bound(array, end, value);
-   MFEM_VERIFY(pos != end, "value not found");
-   return pos - array;
-}
-
 /** Dissect a DOF number to obtain the entity type (0=vertex, 1=edge, 2=face),
  *  entity index and the DOF number within the entity.
  */
@@ -1519,17 +1441,9 @@ void ParFiniteElementSpace::UnpackDof(int dof,
       dof -= nedofs;
       if (dof < nfdofs) // regular face
       {
-         if (fdofs) // have mixed faces
-         {
-            index = bisect(fdofs+1, mesh->GetNFaces(), dof);
-            edof = dof - fdofs[index];
-         }
-         else // uniform faces
-         {
-            int nf = fec->DofForGeometry(pncmesh->GetFaceGeometry(0));
-            index = dof / nf, edof = dof % nf;
-         }
-         entity = 2;
+         MFEM_ASSERT(!pmesh->HasGeometry(Geometry::TRIANGLE), "");
+         int nf = fec->DofForGeometry(Geometry::SQUARE);
+         entity = 2, index = dof / nf, edof = dof % nf;
          return;
       }
       MFEM_ABORT("Cannot unpack internal DOF");
@@ -1553,17 +1467,8 @@ void ParFiniteElementSpace::UnpackDof(int dof,
       dof -= ngedofs;
       if (dof < ngfdofs) // ghost face
       {
-         if (gfdofs) // have mixed faces
-         {
-            index = bisect(gfdofs+1, pncmesh->GetNGhostFaces(), dof);
-            edof = dof - gfdofs[index];
-         }
-         else // uniform faces
-         {
-            int nf = fec->DofForGeometry(pncmesh->GetFaceGeometry(0));
-            index = pncmesh->GetNFaces() + dof / nf, edof = dof % nf;
-         }
-         entity = 2;
+         int nf = fec->DofForGeometry(pncmesh->GetGhostFaceGeometry(0));
+         entity = 2, index = pncmesh->GetNFaces() + dof / nf, edof = dof % nf;
          return;
       }
       MFEM_ABORT("Out of range DOF.");
@@ -1746,7 +1651,7 @@ void NeighborRowMessage::Encode(int rank)
          mfem::out << "Rank " << pncmesh->MyRank << " sending to " << rank
                    << ": ent " << ri.entity << ", index " << ri.index
                    << ", edof " << ri.edof << " (id " << id.element << "/"
-                   << int(id.local) << ")" << std::endl;
+                   << id.local << ")" << std::endl;
 #endif
 
          // handle orientation and sign change
@@ -1789,6 +1694,8 @@ void NeighborRowMessage::Decode(int rank)
    rows.clear();
    rows.reserve(nrows);
 
+   Geometry::Type fgeom = pncmesh->GetFaceGeometry();
+
    // read rows
    for (int ent = 0, gi = 0; ent < 3; ent++)
    {
@@ -1807,9 +1714,8 @@ void NeighborRowMessage::Decode(int rank)
          }
          else if (ent == 2)
          {
-            Geometry::Type geom = pncmesh->GetFaceGeometry(id.index);
             int fo = pncmesh->GetFaceOrientation(id.index);
-            ind = fec->DofOrderForOrientation(geom, fo);
+            ind = fec->DofOrderForOrientation(fgeom, fo);
          }
 
          double s = 1.0;
@@ -1898,7 +1804,7 @@ void ParFiniteElementSpace
    for (int i = 0; i < dof_group.Size(); i++)
    {
       os << i << ": ";
-      if (i < (nvdofs + nedofs + nfdofs) || i >= ndofs)
+      if (i < (nvdofs + nedofs + nfdofs) || i > ndofs)
       {
          int ent, idx, edof;
          UnpackDof(i, ent, idx, edof);
@@ -1980,7 +1886,15 @@ int ParFiniteElementSpace
          if (!list.masters.size()) { continue; }
 
          IsoparametricTransformation T;
-         DenseMatrix I;
+         if (entity > 1) { T.SetFE(&QuadrilateralFE); }
+         else { T.SetFE(&SegmentFE); }
+
+         Geometry::Type geom = (entity > 1) ?
+                               Geometry::SQUARE : Geometry::SEGMENT;
+         const FiniteElement* fe = fec->FiniteElementForGeometry(geom);
+         if (!fe) { continue; }
+
+         DenseMatrix I(fe->GetDof());
 
          // process masters that we own or that affect our edges/faces
          for (unsigned mi = 0; mi < list.masters.size(); mi++)
@@ -1994,17 +1908,6 @@ int ParFiniteElementSpace
 
             if (!master_dofs.Size()) { continue; }
 
-            const FiniteElement* fe = fec->FiniteElementForGeometry(mf.Geom());
-            if (!fe) { continue; }
-
-            switch (mf.Geom())
-            {
-               case Geometry::SQUARE:   T.SetFE(&QuadrilateralFE); break;
-               case Geometry::TRIANGLE: T.SetFE(&TriangleFE); break;
-               case Geometry::SEGMENT:  T.SetFE(&SegmentFE); break;
-               default: MFEM_ABORT("unsupported geometry");
-            }
-
             // constrain slaves that exist in our mesh
             for (int si = mf.slaves_begin; si < mf.slaves_end; si++)
             {
@@ -2055,8 +1958,6 @@ int ParFiniteElementSpace
                   (l == 1) ? (const MeshId&) list.masters[i]
                   /*    */ : (const MeshId&) list.slaves[i];
 
-               if (id.index < 0) { continue; }
-
                GroupId owner = pncmesh->GetEntityOwnerId(entity, id.index);
                GroupId group = pncmesh->GetEntityGroupId(entity, id.index);
 
@@ -2553,9 +2454,6 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    int nrk = HYPRE_AssumedPartitionCheck() ? 2 : NRanks;
 
    MFEM_VERIFY(Nonconforming(), "Not implemented for conforming meshes.");
-   MFEM_VERIFY(pmesh->GetNumGeometries(pmesh->Dimension()) == 1,
-               "Not implemented for mixed meshes.");
-
    MFEM_VERIFY(old_dof_offsets[nrk], "Missing previous (finer) space.");
    MFEM_VERIFY(dof_offsets[nrk] <= old_dof_offsets[nrk],
                "Previous space is not finer.");
@@ -2569,7 +2467,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    Vector row;
 
    ParNCMesh* pncmesh = pmesh->pncmesh;
-   Geometry::Type geom = pncmesh->GetElementGeometry(0); // TODO mixed meshes
+   Geometry::Type geom = pncmesh->GetElementGeometry();
    int ldof = fec->FiniteElementForGeometry(geom)->GetDof();
 
    const CoarseFineTransformations &dtrans = pncmesh->GetDerefinementTransforms();
@@ -2793,8 +2691,6 @@ void ParFiniteElementSpace::Destroy()
    delete Pconf; Pconf = NULL;
    delete R; R = NULL;
 
-   delete [] gfdofs; gfdofs = NULL;
-
    delete gcomm; gcomm = NULL;
 
    num_face_nbr_dofs = -1;
@@ -2967,8 +2863,8 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    MFEM_ASSERT(x.Size() == Width(), "");
    MFEM_ASSERT(y.Size() == Height(), "");
 
-   const double *xdata = x.HostRead();
-   double *ydata = y.HostWrite();
+   const double *xdata = x.GetData();
+   double *ydata = y.GetData();
    const int m = external_ldofs.Size();
 
    const int in_layout = 2; // 2 - input is ltdofs array
@@ -2993,8 +2889,8 @@ void ConformingProlongationOperator::MultTranspose(
    MFEM_ASSERT(x.Size() == Height(), "");
    MFEM_ASSERT(y.Size() == Width(), "");
 
-   const double *xdata = x.HostRead();
-   double *ydata = y.HostWrite();
+   const double *xdata = x.GetData();
+   double *ydata = y.GetData();
    const int m = external_ldofs.Size();
 
    gc.ReduceBegin(xdata);
@@ -3012,242 +2908,6 @@ void ConformingProlongationOperator::MultTranspose(
    gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
 }
 
-DeviceConformingProlongationOperator::DeviceConformingProlongationOperator(
-   const ParFiniteElementSpace &pfes) :
-   ConformingProlongationOperator(pfes),
-   mpi_gpu_aware(Device::GetGPUAwareMPI())
-{
-   MFEM_ASSERT(pfes.Conforming(), "internal error");
-   const SparseMatrix *R = pfes.GetRestrictionMatrix();
-   MFEM_ASSERT(R->Finalized(), "");
-   const int tdofs = R->Height();
-   MFEM_ASSERT(tdofs == pfes.GetTrueVSize(), "");
-   MFEM_ASSERT(tdofs == R->GetI()[tdofs], "");
-   ltdof_ldof = Array<int>(const_cast<int*>(R->GetJ()), tdofs);
-   ltdof_ldof.UseDevice();
-   {
-      Table nbr_ltdof;
-      gc.GetNeighborLTDofTable(nbr_ltdof);
-      const int nb_connections = nbr_ltdof.Size_of_connections();
-      shr_ltdof.SetSize(nb_connections);
-      shr_ltdof.CopyFrom(nbr_ltdof.GetJ());
-      shr_buf.SetSize(nb_connections);
-      shr_buf.UseDevice(true);
-      shr_buf_offsets = nbr_ltdof.GetI();
-      {
-         Array<int> shr_ltdof(nbr_ltdof.GetJ(), nb_connections);
-         Array<int> unique_ltdof(shr_ltdof);
-         unique_ltdof.Sort();
-         unique_ltdof.Unique();
-         // Note: the next loop modifies the J array of nbr_ltdof
-         for (int i = 0; i < shr_ltdof.Size(); i++)
-         {
-            shr_ltdof[i] = unique_ltdof.FindSorted(shr_ltdof[i]);
-            MFEM_ASSERT(shr_ltdof[i] != -1, "internal error");
-         }
-         Table unique_shr;
-         Transpose(shr_ltdof, unique_shr, unique_ltdof.Size());
-         unq_ltdof = Array<int>(unique_ltdof, unique_ltdof.Size());
-         unq_shr_i = Array<int>(unique_shr.GetI(), unique_shr.Size()+1);
-         unq_shr_j = Array<int>(unique_shr.GetJ(), unique_shr.Size_of_connections());
-      }
-      delete [] nbr_ltdof.GetJ();
-      nbr_ltdof.LoseData();
-   }
-   {
-      Table nbr_ldof;
-      gc.GetNeighborLDofTable(nbr_ldof);
-      const int nb_connections = nbr_ldof.Size_of_connections();
-      ext_ldof.SetSize(nb_connections);
-      ext_ldof.CopyFrom(nbr_ldof.GetJ());
-      ext_buf.SetSize(nb_connections);
-      ext_buf.UseDevice(true);
-      ext_buf_offsets = nbr_ldof.GetI();
-      delete [] nbr_ldof.GetJ();
-      nbr_ldof.LoseData();
-   }
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = shr_buf_offsets[nbr];
-      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0) { req_counter++; }
-
-      const int recv_offset = ext_buf_offsets[nbr];
-      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0) { req_counter++; }
-   }
-   requests = new MPI_Request[req_counter];
-}
-
-static void ExtractSubVector(const int N,
-                             const Array<int> &indices,
-                             const Vector &in, Vector &out)
-{
-   auto y = out.Write();
-   const auto x = in.Read();
-   const auto I = indices.Read();
-   MFEM_FORALL(i, N, y[i] = x[I[i]];); // indices can be repeated
-}
-
-void DeviceConformingProlongationOperator::BcastBeginCopy(
-   const Vector &x) const
-{
-   // shr_buf[i] = src[shr_ltdof[i]]
-   if (shr_ltdof.Size() == 0) { return; }
-   ExtractSubVector(shr_ltdof.Size(), shr_ltdof, x, shr_buf);
-   // If the above kernel is executed asynchronously, we should wait for it to
-   // complete
-   if (mpi_gpu_aware) { Device::Synchronize(); }
-}
-
-static void SetSubVector(const int N,
-                         const Array<int> &indices,
-                         const Vector &in, Vector &out)
-{
-   auto y = out.Write();
-   const auto x = in.Read();
-   const auto I = indices.Read();
-   MFEM_FORALL(i, N, y[I[i]] = x[i];);
-}
-
-void DeviceConformingProlongationOperator::BcastLocalCopy(
-   const Vector &x, Vector &y) const
-{
-   // dst[ltdof_ldof[i]] = src[i]
-   if (ltdof_ldof.Size() == 0) { return; }
-   SetSubVector(ltdof_ldof.Size(), ltdof_ldof, x, y);
-}
-
-void DeviceConformingProlongationOperator::BcastEndCopy(
-   Vector &y) const
-{
-   // dst[ext_ldof[i]] = ext_buf[i]
-   if (ext_ldof.Size() == 0) { return; }
-   SetSubVector(ext_ldof.Size(), ext_ldof, ext_buf, y);
-}
-
-void DeviceConformingProlongationOperator::Mult(const Vector &x,
-                                                Vector &y) const
-{
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-   BcastBeginCopy(x); // copy to 'shr_buf'
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = shr_buf_offsets[nbr];
-      const int send_size = shr_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
-      {
-         auto send_buf = mpi_gpu_aware ? shr_buf.Read() : shr_buf.HostRead();
-         MPI_Isend(send_buf + send_offset, send_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41822,
-                   gtopo.GetComm(), &requests[req_counter++]);
-      }
-      const int recv_offset = ext_buf_offsets[nbr];
-      const int recv_size = ext_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         auto recv_buf = mpi_gpu_aware ? ext_buf.Write() : ext_buf.HostWrite();
-         MPI_Irecv(recv_buf + recv_offset, recv_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41822,
-                   gtopo.GetComm(), &requests[req_counter++]);
-      }
-   }
-   BcastLocalCopy(x, y);
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-   BcastEndCopy(y); // copy from 'ext_buf'
-}
-
-DeviceConformingProlongationOperator::~DeviceConformingProlongationOperator()
-{
-   delete [] requests;
-   delete [] ext_buf_offsets;
-   delete [] shr_buf_offsets;
-}
-
-void DeviceConformingProlongationOperator::ReduceBeginCopy(
-   const Vector &x) const
-{
-   // ext_buf[i] = src[ext_ldof[i]]
-   if (ext_ldof.Size() == 0) { return; }
-   ExtractSubVector(ext_ldof.Size(), ext_ldof, x, ext_buf);
-   // If the above kernel is executed asynchronously, we should wait for it to
-   // complete
-   if (mpi_gpu_aware) { Device::Synchronize(); }
-}
-
-void DeviceConformingProlongationOperator::ReduceLocalCopy(
-   const Vector &x, Vector &y) const
-{
-   // dst[i] = src[ltdof_ldof[i]]
-   if (ltdof_ldof.Size() == 0) { return; }
-   ExtractSubVector(ltdof_ldof.Size(), ltdof_ldof, x, y);
-}
-
-static void AddSubVector(const int num_unique_dst_indices,
-                         const Array<int> &unique_dst_indices,
-                         const Array<int> &unique_to_src_offsets,
-                         const Array<int> &unique_to_src_indices,
-                         const Vector &src,
-                         Vector &dst)
-{
-   auto y = dst.Write();
-   const auto x = src.Read();
-   const auto DST_I = unique_dst_indices.Read();
-   const auto SRC_O = unique_to_src_offsets.Read();
-   const auto SRC_I = unique_to_src_indices.Read();
-   MFEM_FORALL(i, num_unique_dst_indices,
-   {
-      const int dst_idx = DST_I[i];
-      double sum = y[dst_idx];
-      const int end = SRC_O[i+1];
-      for (int j = SRC_O[i]; j != end; ++j) { sum += x[SRC_I[j]]; }
-      y[dst_idx] = sum;
-   });
-}
-
-void DeviceConformingProlongationOperator::ReduceEndAssemble(Vector &y) const
-{
-   // dst[shr_ltdof[i]] += shr_buf[i]
-   const int unq_ltdof_size = unq_ltdof.Size();
-   if (unq_ltdof_size == 0) { return; }
-   AddSubVector(unq_ltdof_size, unq_ltdof, unq_shr_i, unq_shr_j, shr_buf, y);
-}
-
-void DeviceConformingProlongationOperator::MultTranspose(const Vector &x,
-                                                         Vector &y) const
-{
-   const GroupTopology &gtopo = gc.GetGroupTopology();
-   ReduceBeginCopy(x); // copy to 'ext_buf'
-   int req_counter = 0;
-   for (int nbr = 1; nbr < gtopo.GetNumNeighbors(); nbr++)
-   {
-      const int send_offset = ext_buf_offsets[nbr];
-      const int send_size = ext_buf_offsets[nbr+1] - send_offset;
-      if (send_size > 0)
-      {
-         auto send_buf = mpi_gpu_aware ? ext_buf.Read() : ext_buf.HostRead();
-         MPI_Isend(send_buf + send_offset, send_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41823,
-                   gtopo.GetComm(), &requests[req_counter++]);
-      }
-      const int recv_offset = shr_buf_offsets[nbr];
-      const int recv_size = shr_buf_offsets[nbr+1] - recv_offset;
-      if (recv_size > 0)
-      {
-         auto recv_buf = mpi_gpu_aware ? shr_buf.Write() : shr_buf.HostWrite();
-         MPI_Irecv(recv_buf + recv_offset, recv_size, MPI_DOUBLE,
-                   gtopo.GetNeighborRank(nbr), 41823,
-                   gtopo.GetComm(), &requests[req_counter++]);
-      }
-   }
-   ReduceLocalCopy(x, y);
-   MPI_Waitall(req_counter, requests, MPI_STATUSES_IGNORE);
-   ReduceEndAssemble(y); // assemble from 'shr_buf'
-}
-
 } // namespace mfem
 
 #endif

fem/pfespace.hpp

@@ -46,7 +46,6 @@ private:
 
    /// Number of vertex/edge/face/total ghost DOFs (nonconforming case).
    int ngvdofs, ngedofs, ngfdofs, ngdofs;
-   int* gfdofs;
 
    /// The group of each local dof.
    Array<int> ldof_group;
@@ -114,7 +113,7 @@ private:
    void GetGhostFaceDofs(const MeshId &face_id, Array<int> &dofs) const;
 
    void GetGhostDofs(int entity, const MeshId &id, Array<int> &dofs) const;
-   /// Return the dofs associated with the interior of the given mesh entity.
+   // Return the dofs associated with the interior of the given mesh entity.
    void GetBareDofs(int entity, int index, Array<int> &dofs) const;
 
    int  PackDof(int entity, int index, int edof) const;
@@ -388,52 +387,6 @@ public:
    virtual void MultTranspose(const Vector &x, Vector &y) const;
 };
 
-/// Auxiliary device class used by ParFiniteElementSpace.
-class DeviceConformingProlongationOperator: public
-   ConformingProlongationOperator
-{
-protected:
-   bool mpi_gpu_aware;
-   Array<int> shr_ltdof, ext_ldof;
-   mutable Vector shr_buf, ext_buf;
-   int *shr_buf_offsets, *ext_buf_offsets;
-   Array<int> ltdof_ldof, unq_ltdof;
-   Array<int> unq_shr_i, unq_shr_j;
-   MPI_Request *requests;
-   // Kernel: copy ltdofs from 'src' to 'shr_buf' - prepare for send.
-   //         shr_buf[i] = src[shr_ltdof[i]]
-   void BcastBeginCopy(const Vector &src) const;
-
-   // Kernel: copy ltdofs from 'src' to ldofs in 'dst'.
-   //         dst[ltdof_ldof[i]] = src[i]
-   void BcastLocalCopy(const Vector &src, Vector &dst) const;
-
-   // Kernel: copy ext. dofs from 'ext_buf' to 'dst' - after recv.
-   //         dst[ext_ldof[i]] = ext_buf[i]
-   void BcastEndCopy(Vector &dst) const;
-
-   // Kernel: copy ext. dofs from 'src' to 'ext_buf' - prepare for send.
-   //         ext_buf[i] = src[ext_ldof[i]]
-   void ReduceBeginCopy(const Vector &src) const;
-
-   // Kernel: copy owned ldofs from 'src' to ltdofs in 'dst'.
-   //         dst[i] = src[ltdof_ldof[i]]
-   void ReduceLocalCopy(const Vector &src, Vector &dst) const;
-
-   // Kernel: assemble dofs from 'shr_buf' into to 'dst' - after recv.
-   //         dst[shr_ltdof[i]] += shr_buf[i]
-   void ReduceEndAssemble(Vector &dst) const;
-
-public:
-   DeviceConformingProlongationOperator(const ParFiniteElementSpace &pfes);
-
-   virtual ~DeviceConformingProlongationOperator();
-
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual void MultTranspose(const Vector &x, Vector &y) const;
-};
-
 }
 
 #endif // MFEM_USE_MPI

fem/pgridfunc.cpp

@@ -225,13 +225,11 @@ void ParGridFunction::ExchangeFaceNbrData()
    MPI_Request *recv_requests = requests + num_face_nbrs;
    MPI_Status  *statuses = new MPI_Status[num_face_nbrs];
 
-   const double *h_data = this->HostRead();
    for (int i = 0; i < send_data.Size(); i++)
    {
-      send_data[i] = h_data[send_ldof[i]];
+      send_data[i] = data[send_ldof[i]];
    }
 
-   double *h_face_nbr_data = face_nbr_data.HostWrite();
    for (int fn = 0; fn < num_face_nbrs; fn++)
    {
       int nbr_rank = pmesh->GetFaceNbrRank(fn);
@@ -241,7 +239,7 @@ void ParGridFunction::ExchangeFaceNbrData()
                 send_offset[fn+1] - send_offset[fn],
                 MPI_DOUBLE, nbr_rank, tag, MyComm, &send_requests[fn]);
 
-      MPI_Irecv(&h_face_nbr_data[recv_offset[fn]],
+      MPI_Irecv(&face_nbr_data(recv_offset[fn]),
                 recv_offset[fn+1] - recv_offset[fn],
                 MPI_DOUBLE, nbr_rank, tag, MyComm, &recv_requests[fn]);
    }
@@ -369,10 +367,10 @@ void ParGridFunction::ProjectDiscCoefficient(Coefficient &coeff, AvgType type)
    GroupCommunicator &gcomm = pfes->GroupComm();
    gcomm.Reduce<int>(zones_per_vdof, GroupCommunicator::Sum);
    gcomm.Bcast(zones_per_vdof);
-
-   // Accumulate for all vdofs.
-   gcomm.Reduce<double>(data, GroupCommunicator::Sum);
-   gcomm.Bcast<double>(data);
+   // Accumulate for all tdofs.
+   HypreParVector *tv = this->ParallelAssemble();
+   this->Distribute(tv);
+   delete tv;
 
    ComputeMeans(type, zones_per_vdof);
 }
@@ -391,10 +389,10 @@ void ParGridFunction::ProjectDiscCoefficient(VectorCoefficient &vcoeff,
    GroupCommunicator &gcomm = pfes->GroupComm();
    gcomm.Reduce<int>(zones_per_vdof, GroupCommunicator::Sum);
    gcomm.Bcast(zones_per_vdof);
-
-   // Accumulate for all vdofs.
-   gcomm.Reduce<double>(data, GroupCommunicator::Sum);
-   gcomm.Bcast<double>(data);
+   // Accumulate for all tdofs.
+   HypreParVector *tv = this->ParallelAssemble();
+   this->Distribute(tv);
+   delete tv;
 
    ComputeMeans(type, zones_per_vdof);
 }
@@ -427,8 +425,8 @@ void ParGridFunction::ProjectBdrCoefficient(
    }
    else
    {
-      // TODO: is this the same as the conforming case (after the merge of
-      //       cut-mesh-groups-dev)?
+      // FIXME: same as the conforming case after 'cut-mesh-groups-dev-*' is
+      //        merged?
       ComputeMeans(ARITHMETIC, values_counter);
    }
 #ifdef MFEM_DEBUG
@@ -471,8 +469,8 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
    }
    else
    {
-      // TODO: is this the same as the conforming case (after the merge of
-      //       cut-mesh-groups-dev)?
+      // FIXME: same as the conforming case after 'cut-mesh-groups-dev-*' is
+      //        merged?
       ComputeMeans(ARITHMETIC, values_counter);
    }
 #ifdef MFEM_DEBUG
@@ -489,17 +487,16 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 
 void ParGridFunction::Save(std::ostream &out) const
 {
-   double *data_  = const_cast<double*>(HostRead());
    for (int i = 0; i < size; i++)
    {
-      if (pfes->GetDofSign(i) < 0) { data_[i] = -data_[i]; }
+      if (pfes->GetDofSign(i) < 0) { data[i] = -data[i]; }
    }
 
    GridFunction::Save(out);
 
    for (int i = 0; i < size; i++)
    {
-      if (pfes->GetDofSign(i) < 0) { data_[i] = -data_[i]; }
+      if (pfes->GetDofSign(i) < 0) { data[i] = -data[i]; }
    }
 }
 

fem/pgridfunc.hpp

@@ -112,8 +112,6 @@ public:
    /// Associate a new parallel space with the ParGridFunction.
    void SetSpace(ParFiniteElementSpace *f);
 
-   using GridFunction::MakeRef;
-
    /** @brief Make the ParGridFunction reference external data on a new
        FiniteElementSpace. */
    /** This method changes the FiniteElementSpace associated with the

fem/pnonlinearform.cpp

@@ -46,7 +46,7 @@ double ParNonlinearForm::GetParGridFunctionEnergy(const Vector &x) const
 void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
    NonlinearForm::Mult(x, y); // x --(P)--> aux1 --(A_local)--> aux2
-   Y.MakeRef(aux2, 0); // aux2 contains A_local.P.x
+   Y.SetData(aux2.GetData()); // aux2 contains A_local.P.x
 
    if (fnfi.Size())
    {
@@ -58,7 +58,7 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
       Array<int> vdofs1, vdofs2;
       Vector el_x, el_y;
 
-      X.MakeRef(aux1, 0); // aux1 contains P.x
+      X.SetData(aux1.GetData()); // aux1 contains P.x
       X.ExchangeFaceNbrData();
       const int n_shared_faces = pmesh->GetNSharedFaces();
       for (int i = 0; i < n_shared_faces; i++)

fem/sidredatacollection.cpp

@@ -16,7 +16,9 @@
 
 #include "fem.hpp"
 
-#include <axom/sidre.hpp>
+#ifdef MFEM_USE_MPI
+#include <sidre/IOManager.hpp>
+#endif
 
 #include <string>
 #include <iomanip>      // for setw, setfill
@@ -202,10 +204,10 @@ SidreDataCollection::get_file_path(const std::string &filename) const
 
 axom::sidre::View *
 SidreDataCollection::AllocNamedBuffer(const std::string& buffer_name,
-                                      axom::sidre::IndexType sz,
+                                      axom::sidre::SidreLength sz,
                                       axom::sidre::TypeID type)
 {
-   sz = std::max(sz, sidre::IndexType(0));
+   sz = std::max(sz, sidre::SidreLength(0));
    sidre::Group *f = named_buffers_grp();
    sidre::View  *v = NULL;
 
@@ -823,7 +825,7 @@ void SidreDataCollection::Save(const std::string& filename,
 void SidreDataCollection::
 addScalarBasedGridFunction(const std::string &field_name, GridFunction *gf,
                            const std::string &buffer_name,
-                           axom::sidre::IndexType offset)
+                           axom::sidre::SidreLength offset)
 {
    sidre::Group* grp = m_bp_grp->getGroup("fields/" + field_name);
    MFEM_ASSERT(grp != NULL, "field " << field_name << " does not exist");
@@ -886,7 +888,7 @@ addScalarBasedGridFunction(const std::string &field_name, GridFunction *gf,
 void SidreDataCollection::
 addVectorBasedGridFunction(const std::string& field_name, GridFunction *gf,
                            const std::string &buffer_name,
-                           axom::sidre::IndexType offset)
+                           axom::sidre::SidreLength offset)
 {
    sidre::Group* grp = m_bp_grp->getGroup("fields/" + field_name);
    MFEM_ASSERT(grp != NULL, "field " << field_name << " does not exist");
@@ -1011,7 +1013,7 @@ DeregisterFieldInBPIndex(const std::string& field_name)
 void SidreDataCollection::RegisterField(const std::string &field_name,
                                         GridFunction *gf,
                                         const std::string &buffer_name,
-                                        axom::sidre::IndexType offset)
+                                        axom::sidre::SidreLength offset)
 {
    if ( field_name.empty() || buffer_name.empty() ||
         gf == NULL || gf->FESpace() == NULL )

fem/sidredatacollection.hpp

@@ -25,7 +25,7 @@
 #  pragma GCC diagnostic ignored "-Wpedantic"
 # endif
 #endif
-#include <axom/sidre.hpp>
+#include <sidre/sidre.hpp>
 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
 # pragma GCC diagnostic pop
 #endif
@@ -246,7 +246,7 @@ public:
     */
    void RegisterField(const std::string &field_name, GridFunction *gf,
                       const std::string &buffer_name,
-                      axom::sidre::IndexType offset);
+                      axom::sidre::SidreLength offset);
 
    /// Registers an attribute field in the Sidre DataStore
    /** The registration process is similar to that of RegisterField()
@@ -385,7 +385,7 @@ public:
     */
    axom::sidre::View *
    AllocNamedBuffer(const std::string& buffer_name,
-                    axom::sidre::IndexType sz,
+                    axom::sidre::SidreLength sz,
                     axom::sidre::TypeID type =
                        axom::sidre::DOUBLE_ID);
 
@@ -469,7 +469,7 @@ private:
    void addScalarBasedGridFunction(const std::string& field_name,
                                    GridFunction* gf,
                                    const std::string &buffer_name,
-                                   axom::sidre::IndexType offset);
+                                   axom::sidre::SidreLength offset);
 
    /**
     * \brief A private helper function to set up the views associated with the
@@ -483,7 +483,7 @@ private:
    void addVectorBasedGridFunction(const std::string& field_name,
                                    GridFunction* gf,
                                    const std::string &buffer_name,
-                                   axom::sidre::IndexType offset);
+                                   axom::sidre::SidreLength offset);
 
    /** @brief A private helper function to set up the Views associated with
        attribute field named @a field_name */