Merge remote-tracking branch 'origin/master' into complex-strumpack-dev

# Conflicts:
#	examples/ex11p.cpp
#	linalg/strumpack.cpp
#	linalg/strumpack.hpp

.appveyor.yml

@@ -26,25 +26,24 @@ install:
 - cd ..
 
 # Install hypre
-- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
-- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
-- cd hypre-2-10-0b
-- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz'
+- 7z x hypre-2.10.0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2.10.0b
+- cmake -Hsrc -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+# - cmake -Hsrc -Bbuild -DCMAKE_BUILD_TYPE=Release -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
 - cd ..
 
 # MFEM
 before_build:
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 
 build_script:
 - cmake --build build_parallel
 - cmake --build build_serial
-- cmake --build build_serial --target exec
 
 after_build:
 # - cmake --build build_parallel --target check
-- cmake --build build_serial --target RUN_TESTS
-
+- cmake --build build_serial --target check

.gitignore

@@ -6,8 +6,6 @@
 # Also ignore OS-specific files like .DS_Store on Mac
 # ------------------------------------------------------------------------------
 
-*DS_Store
-
 # Object and library files
 *.o
 /libmfem.*
@@ -26,7 +24,6 @@ CMakeFiles/
 /deps.mk
 config/_config.hpp
 config/config.mk
-config/user.cmake
 config/sample-runs-build.log
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
@@ -85,9 +82,9 @@ examples/ex20.dat
 examples/ex20p_?????.dat
 examples/gnuplot_ex20.inp
 examples/gnuplot_ex20p.inp
-examples/ex21*.mesh
-examples/ex21*.sol
-examples/ex21p_*.*
+examples/ex22*.mesh
+examples/ex22*.sol
+examples/ex22p_*.*
 
 examples/sundials/ex9
 examples/sundials/ex1[06]
@@ -186,6 +183,3 @@ miniapps/nurbs/Example1*
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
-
-# VPATH builds
-build-*/*

.travis.yml

@@ -205,7 +205,6 @@ install:
         else
             echo "Reusing cached hypre-2.10.0b/";
         fi;
-        ln -s hypre-2.10.0b hypre;
      else
         echo "Serial build, not using hypre";
      fi

CHANGELOG

@@ -8,21 +8,23 @@
                                http://mfem.org
 
 
-Version 4.0.1 (development)
-===========================
+Version 4.0-RC1, Apr 11, 2019
+=============================
 
-GPU support
------------
-- Added initial support for AMD GPUs based on HIP: a C++ runtime API and kernel
-  language that can run on both AMD and NVIDIA hardware. The list of current
-  backends is: "occa-cuda", "raja-cuda", "cuda", "hip", "occa-omp", "raja-omp",
-  "omp", "occa-cpu", "raja-cpu", and "cpu".
+Requirements and Limitations
+----------------------------
+- This is a release candidate for mfem-4.0.
+- Use at your own risk -- not everything will work, the API may change.
+- We are looking for feedback from friendly users.
+- Unlike previous MFEM releases, this version requires a C++11 compiler.
 
-
-Version 4.0, released on May 24, 2019
-=====================================
-
-Unlike previous MFEM releases, this version requires a C++11 compiler.
+- GPU-related limitations:
+  * NVCC is not supported in the CMake build system yet.
+  * Element batching is currently ignored.
+  * Full-assembly (on device), element assembly, and matrix-free bilinear forms
+    are not supported yet.
+  * FunctionCoefficients do not currently work on GPUs.
+  * Partial assembly kernels are not implemented yet for simplices.
 
 GPU support
 -----------
@@ -33,7 +35,7 @@ GPU support
   seamlessly with a new lightweight device/host memory manager. The kernels can
   be implemented either in OCCA, or as a simple wrapper around for-loops, which
   can then be dispatched to RAJA and native backends. See the files forall.hpp
-  and mem_manager.hpp in the general/ directory for more details.
+  and mem_manager.hpp in the general/ directory.
 
 - Several of the MFEM example codes (ex1, ex1p, ex6, and ex6p) can now take
   advantage of GPU acceleration with the backend selectable at runtime. Many of
@@ -41,44 +43,26 @@ GPU support
   bilinear forms) have been extended to take advantage of kernel acceleration by
   simply replacing loops with the MFEM_FORALL() macro.
 
-- In addition to native CUDA kernels, the library currently supports OCCA, RAJA
-  and OpenMP kernels, which could be mixed and matched in different parts of the
-  same application. We plan on adding support for more programming models and
-  devices in the future, without the need for significant modifications in user
-  code. The list of current backends is: "occa-cuda", "raja-cuda", "cuda",
-  "occa-omp", "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".
-
-- GPU-related limitations:
-  * Hypre preconditioners are not yet available in GPU mode, and in particular
-    hypre must be built in CPU mode.
-  * Only constant coefficients are currently supported on GPUs.
-  * Optimized element assembly, and matrix-free bilinear forms are not
-    implemented yet. Element batching is currently ignored.
-  * In device mode, full assembly is performed on the host (but the matvec
-    action is performed on the device).
-  * Partial assembly kernels are not implemented yet for simplices.
+- In addition to pure CUDA, the library currently supports OCCA, RAJA and OpenMP
+  kernels, which could be mixed and matched in different parts of the same
+  application. We plan on adding support for more programming models and devices
+  in the future, without the need for significant modifications in user code.
+  The list of current backends is: "occa-cuda", "raja-cuda", "cuda", "occa-omp",
+  "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".
 
 Discretization improvements
 ---------------------------
-
-- Partial assembled finite element operators are now available in the core
-  library, based on the new classes PABilinearFormExtension, ElementRestriction,
-  DofToQuad and GeometricFactors (associated with the classes BilinearForm,
-  FiniteElementSpace, FiniteElement and Mesh, respectively). The kernels for
-  partial assembled Setup/Assembly and Action/Mult are implemented in the
-  BilinearFormIntegrator methods AssemblePA and AddMultPA.
-
 - Added support for a general "low-order refined"-to-"high-order" transfer of
   GridFunction data from a "low-order refined" (LOR) space defined on a refined
   mesh to a "high-order" (HO) finite element space defined on a coarse mesh. See
   the new classes InterpolationGridTransfer and L2ProjectionGridTransfer and the
   new LOR Transfer miniapp: miniapps/tools/lor-transfer.cpp.
 
+- Added support for derefinement of vector (RT + ND) spaces.
+
 - Added element flux, and flux energy computation in class ElasticityIntegrator,
   allowing for the use of Zienkiewicz-Zhu type error estimators with the
-  integrator. For an illustration of this addition, see the new Example 21.
-
-- Added support for derefinement of vector (RT + ND) spaces.
+  integrator. For an illustration of this addition, see the new Example 22.
 
 - Added a variety of coefficients which are sums or products of existing
   coefficients as well as grid function coefficients which return the
@@ -90,13 +74,13 @@ Support for wedge elements and meshes with mixed element types
   type PRISM) which have two triangular faces and three quadrilateral faces.
   Several examples of such meshes can be found in the data/ directory.
 
+- Added H1 and L2 finite elements of arbitrary order for Wedge elements.
+
 - Added support for mixed meshes containing triangles and quadrilaterals in 2D
   or tetrahedra, wedges, and hexahedra in 3D. This includes support for uniform
   refinement of such meshes. Several examples of such meshes can be found in the
   data/ directory.
 
-- Added H1 and L2 finite elements of arbitrary order for Wedge elements.
-
 - Added support for reading and writing linear and quadratic meshes containing
   wedge elements in VTK mesh format. Several examples of such meshes can be
   found in the data/ directory.
@@ -117,10 +101,6 @@ Other meshing improvements
   This guarantees that the shape regularity of the elements will be preserved
   under refinement.
 
-- The TMOP mesh optimization algorithms were extended to support user-defined
-  space-dependent limiting terms. Improved the TMOP objective functions by more
-  accurate normalization of the different terms.
-
 - Added support for parallel communication groups on non-conforming meshes.
 
 - Improved parallel partitioning of non-conforming meshes. If the coarse mesh
@@ -134,6 +114,10 @@ Other meshing improvements
 - Added support for reading linear and quadratic 2D quadrilateral and triangular
   Cubit meshes.
 
+- The TMOP mesh optimization algorithms were extended to support user-defined
+  space-dependent limiting terms. Improved the TMOP objective functions by more
+  accurate normalization of the different terms.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new meshing miniapp, Toroid, which can produce a variety of torus
@@ -149,7 +133,7 @@ New and updated examples and miniapps
   from a Hamiltonian. The example demonstrates the use of the variable order,
   symplectic integration algorithm implemented in class SIAVSolver.
 
-- Added a new example, Example 21/21p, that illustrates the use of AMR to solve
+- Added a new example, Example 22/22p, that illustrates the use of AMR to solve
   a linear elasticity problem. This is an extension of Example 2/2p.
 
 New and improved solvers and preconditioners
@@ -161,24 +145,17 @@ New and improved solvers and preconditioners
 
 Miscellaneous
 -------------
-- Added unit tests based on the Catch++ library in the test/ directory.
+- Added unit tests based on the Catch++ library.
 
 - Renamed the option MFEM_USE_OPENMP to MFEM_USE_LEGACY_OPENMP. This legacy
   option is deprecated and planned for removal in a future release. The original
   option name, MFEM_USE_OPENMP, is now used to enable the new OpenMP backends in
   the new kernels.
 
-- In SparseMatrix added the option to perform MultTranspose() by matvec with
-  computed and stored transpose matrix. This is required for deterministic
-  results when using devices such as CUDA and OpenMP.
-
 - Altered the way FGMRES counts its iterations so that it matches GMRES.
 
 - Various other simplifications, extensions, and bugfixes in the code.
 
-- Construct abstract parallel rectangular truedof-to-truedof operators via
-  Operator::FormDiscreteOperator().
-
 API changes
 -----------
 - In multiple places, use Geometry::Type instead of int, where appropriate.

CMakeLists.txt

@@ -50,7 +50,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.0.1)
+set(${PROJECT_NAME}_VERSION 3.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -86,13 +86,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 
 # Enable languages.
 enable_language(CXX)
-if (MFEM_USE_CUDA)
-   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
-   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
-   enable_language(CUDA)
-   message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
-endif()
-
 if (XSDK_ENABLE_C)
    enable_language(C)
 endif()
@@ -273,31 +266,6 @@ if (MFEM_USE_PUMI)
   endif()
 endif()
 
-# CUDA
-if (MFEM_USE_CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-   set(CMAKE_CUDA_EXTENSIONS OFF)
-   set(CMAKE_CUDA_FLAGS "-arch=${CUDA_ARCH} --expt-extended-lambda"
-       CACHE STRING "CUDA flags set for MFEM" FORCE)
-   if (MFEM_USE_MPI)
-      set(CUDA_CCBIN_COMPILER ${MPI_CXX_COMPILER})
-   else()
-      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
-   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
-endif()
-
-# OCCA
-if (MFEM_USE_OCCA)
-   find_package(OCCA REQUIRED)
-endif()
-
-# RAJA
-if (MFEM_USE_RAJA)
-   find_package(RAJA REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -323,7 +291,7 @@ endif()
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
     MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
-    POSIXCLOCKS MFEMBacktrace ZLIB OCCA RAJA)
+    POSIXCLOCKS MFEMBacktrace ZLIB)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -359,13 +327,6 @@ set(MFEM_SOURCE_DIRS general linalg mesh fem)
 foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
   add_subdirectory(${DIR})
 endforeach()
-
-if (MFEM_USE_CUDA)
-   foreach(file IN LISTS SOURCES)
-      set_property(SOURCE ${file} PROPERTY LANGUAGE CUDA)
-   endforeach()
-endif()
-
 add_subdirectory(config)
 set(MASTER_HEADERS
   ${PROJECT_SOURCE_DIR}/mfem.hpp
@@ -376,11 +337,6 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON CACHE BOOL "")
 set(CMAKE_INSTALL_RPATH "${_lib_path}" CACHE PATH "")
 set(CMAKE_INSTALL_NAME_DIR "${_lib_path}" CACHE PATH "")
 
-set(MFEM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH
-    "The MFEM source directory" FORCE)
-set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
-    "The MFEM install directory" FORCE)
-
 # Declaring the library
 add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
@@ -395,11 +351,11 @@ endif()
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 
-# If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
-# inside the build directory.
+# If building out-of-source, define MFEM_BUILD_DIR to point to the build
+# directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
   target_compile_definitions(mfem PRIVATE
-    "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
+    "MFEM_BUILD_DIR=${PROJECT_BINARY_DIR}")
 endif()
 
 # Generate configuration file in the build directory: config/_config.hpp.
@@ -415,7 +371,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
       "Writing substitute header --> \"${Header}\"")
     file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
 "// Auto-generated file.
-#define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
+#define MFEM_BUILD_DIR ${PROJECT_BINARY_DIR}
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
     # This version will be installed in the top include directory:
@@ -478,12 +434,12 @@ endif()
 # Add 'check' target - quick test
 if (NOT MFEM_USE_MPI)
   add_custom_target(check
-    ${CMAKE_CTEST_COMMAND} -R \"^ex1_ser\" -C ${CMAKE_CFG_INTDIR}
+    ${CMAKE_CTEST_COMMAND} -R '^ex1_ser' -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
   add_dependencies(check ex1)
 else()
   add_custom_target(check
-    ${CMAKE_CTEST_COMMAND} -R \"^ex1p\" -C ${CMAKE_CFG_INTDIR}
+    ${CMAKE_CTEST_COMMAND} -R '^ex1p' -C ${CMAKE_CFG_INTDIR}
     USES_TERMINAL)
   add_dependencies(check ex1p)
 endif()
@@ -528,13 +484,6 @@ install(DIRECTORY ${MFEM_SOURCE_DIRS}
   DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
   FILES_MATCHING PATTERN "*.hpp")
 
-# Install the okl files
-if (MFEM_USE_OCCA)
-  install(DIRECTORY ${MFEM_SOURCE_DIRS}
-    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "*.okl")
-endif()
-
 # Install ${HEADERS}
 # ---
 # foreach (HDR ${HEADERS})

CONTRIBUTING.md

@@ -142,16 +142,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
   + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
   + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)
 
-- GPU and multi-core CPU support is based on device kernels supporting different
-  backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
-  device/host memory manager.
-
-- The main device-relevant classes and sources are:
-  + [`Device`](http://mfem.github.io/doxygen/html/device_8hpp.html)
-  + [`MemoryManager`](http://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](http://mfem.github.io/doxygen/html/forall_8hpp.html) macro
-  + the [`cuda.hpp`](http://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](http://mfem.github.io/doxygen/html/occa_8hpp.html) files
-
 - The `general/` directory contains C++ classes that serve as utilities for
   communication, error handling, arrays, (Boolean) tables, timing, etc.
 

INSTALL

@@ -13,31 +13,22 @@ of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
 requires an MPI C++ compiler, as well as the following external libraries:
 
 - hypre (a library of high-performance preconditioners)
-  https://github.com/hypre-space/hypre
+  http://www.llnl.gov/CASC/hypre
 
 - METIS (a family of multilevel partitioning algorithms)
   http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
 
-The hypre dependency can be downloaded as a tarball from GitHub or from the
-project webpage https://www.llnl.gov/casc/hypre. For example, the 2.16.0 release
-of hypre is available at
-
-   https://github.com/hypre-space/hypre/archive/v2.16.0.tar.gz
-
 The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.
 
 MFEM also includes support for devices such as GPUs, and programming models such
-as CUDA, HIP, OCCA, OpenMP and RAJA.
+as CUDA, OCCA, OpenMP and RAJA.
 
 - Starting with version 4.0, MFEM requires a C++11 compiler
 
 - CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
   https://developer.nvidia.com/cuda-toolkit
 
-- HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
-
 - OCCA support requires the OCCA library
   https://libocca.org
 
@@ -57,7 +48,7 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
+- Homebrew/Science, https://github.com/Homebrew/homebrew-science
 
 We also recommend downloading and building the MFEM-based GLVis visualization
 tool which can be used to visualize the meshes and solution in MFEM's examples
@@ -69,19 +60,15 @@ Serial build:
    make serial -j 4
 
 Parallel build:
-   (download hypre and METIS 4 from above URLs)
+   (download hypre 2.10.0b and METIS 4 from above URLs)
    (build METIS 4 in ../metis-4.0 relative to mfem/)
-   (build hypre in ../hypre relative to mfem/)
+   (build hypre 2.10.0b in ../hypre-2.10.0b relative to mfem/)
    make parallel -j 4
 
 CUDA build:
    make cuda -j 4
    (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')
 
-HIP build:
-   make hip -j 4
-   (build for a specific AMD GPU chip: 'make hip -j 4 HIP_ARCH=gfx900')
-
 Example codes (serial/parallel, depending on the build):
    cd examples
    make -j 4
@@ -100,19 +87,13 @@ Serial build:
    make -j 4   (assuming "UNIX Makefiles" generator)
 
 Parallel build:
-   (download hypre and METIS 4 from above URLs)
+   (download hypre 2.10.0b and METIS 4 from above URLs)
    (build METIS 4 in ../metis-4.0 relative to mfem/)
-   (build hypre in ../hypre relative to mfem/)
+   (build hypre 2.10.0b in ../hypre-2.10.0b relative to mfem/)
    mkdir <mfem-build-dir> ; cd <mfem-build-dir>
    cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
    make -j 4
 
-CUDA build:
-   (this build requires CMake 3.8 or newer)
-   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_CUDA=YES
-   make -j 4
-
 Example codes (serial/parallel, depending on the build):
    make examples -j 4
 
@@ -168,18 +149,14 @@ Note that re-configuration is only needed to change the currently configured
 options. Several shortcut targets combining (re-)configuration and compilation
 are also defined:
 
-   make serial    -> Builds serial optimized version of the library
-   make parallel  -> Builds parallel optimized version of the library
-   make debug     -> Builds serial debug version of the library
-   make pdebug    -> Builds parallel debug version of the library
-   make cuda      -> Builds serial cuda optimized version of the library
-   make pcuda     -> Builds parallel cuda optimized version of the library
-   make cudebug   -> Builds serial cuda debug version of the library
-   make pcudebug  -> Builds parallel cuda debug version of the library
-   make hip       -> Builds serial hip optimized version of the library
-   make phip      -> Builds parallel hip optimized version of the library
-   make hipdebug  -> Builds serial hip debug version of the library
-   make phipdebug -> Builds parallel hip debug version of the library
+   make serial   -> Builds serial optimized version of the library
+   make parallel -> Builds parallel optimized version of the library
+   make debug    -> Builds serial debug version of the library
+   make pdebug   -> Builds parallel debug version of the library
+   make cuda     -> Builds serial cuda optimized version of the library
+   make pcuda    -> Builds parallel cuda optimized version of the library
+   make cudebug  -> Builds serial cuda debug version of the library
+   make pcudebug -> Builds parallel cuda debug version of the library
 
 Note that any of the above shortcuts accept configuration options, either at the
 command line or through a user configuration file.
@@ -301,7 +278,6 @@ MFEM_THREAD_SAFE = YES/NO
 
 MFEM_USE_LEGACY_OPENMP = YES/NO
    Enable (basic) experimental OpenMP support. Requires MFEM_THREAD_SAFE.
-   This option is deprecated.
 
 MFEM_USE_OPENMP = YES/NO
    Enable the OpenMP backend.
@@ -415,32 +391,28 @@ MFEM_USE_PUMI = YES/NO
    models and effectively supports automated adaptive analysis. PUMI enables
    support for parallel unstructured mesh modifications in MFEM.
 
+MFEM_USE_MM = YES/NO
+   Enables support for the MFEM's memory manager (MM), which is required to
+   support devices with different memory spaces.
+
 MFEM_USE_CUDA = YES/NO
    Enables support for CUDA devices in MFEM. CUDA is a parallel computing
    platform and programming model for general computing on graphical processing
-   units (GPUs). The variable CUDA_ARCH is used to specify the CUDA compute
-   capability used during compilation (by default, CUDA_ARCH=sm_60). When
-   enabled, this option uses the CUDA_* build options, see below.
-
-MFEM_USE_HIP = YES/NO
-   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
-   interface for portability developed by AMD that can target both AMD and
-   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
-   option uses the HIP_* build options, see below.
+   units (GPUs). This option requires MFEM_USE_MM. The variable CUDA_ARCH is
+   used to specify the CUDA compute capability used during compilation (by
+   default, CUDA_ARCH=sm_60). When enabled, this option uses the CUDA_* build
+   options, see below.
 
 MFEM_USE_RAJA = YES/NO
    Enable support for the RAJA performance portability layer in MFEM. RAJA
    provides a portable abstraction for loops, supporting different programming
-   model backends. When using RAJA built with CUDA support, CUDA support must be
-   also enabled in MFEM, i.e. MFEM_USE_CUDA=YES must be set.
+   model backends. When using the RAJA CUDA backend, MFEM_USE_MM is required.
 
 MFEM_USE_OCCA = YES/NO
    Enables support for the OCCA library in MFEM. OCCA is an open-source library
    which aims to make it easy to program different types of devices (e.g. CPU,
    GPU, FPGA) by providing an unified API for interacting with JIT-compiled
-   backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
-   in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.
+   backends. When using the OCCA CUDA backend, MFEM_USE_MM is required.
 
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
@@ -463,7 +435,7 @@ directory and use the string @MFEM_DIR@, e.g. HYPRE_OPT = -I@MFEM_DIR@/../hypre.
 The specific libraries and their options are:
 
 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
-  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
+  URL: http://www.llnl.gov/CASC/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
@@ -561,10 +533,6 @@ The specific libraries and their options are:
   URL: https://developer.nvidia.com/cuda-toolkit
   Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
 
-- HIP, used when MFEM_USE_HIP = YES.
-  URL: https://rocm.github.io/ROCmInstall.html
-  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
-
 - OCCA, used when MFEM_USE_OCCA = YES.
   URL: https://libocca.org
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
@@ -677,8 +645,6 @@ Configuration variables (CMake)
 ===============================
 See the configuration file config/defaults.cmake for the default settings.
 
-Note: the option MFEM_USE_CUDA requires CMake version 3.8 or newer!
-
 Non-standard CMake variables for compilers:
    CXX    - If set, overwrite the auto-detected C++ compiler, serial build
    MPICXX - If set, overwrite the auto-detected MPI C++ compiler, parallel build
@@ -709,9 +675,13 @@ MFEM_USE_NETCDF
 MFEM_USE_MPFR
 MFEM_USE_GZSTREAM
 MFEM_USE_PUMI
+
+The following GNU make options are not supported with CMake yet:
+
 MFEM_USE_CUDA
 MFEM_USE_OCCA
 MFEM_USE_RAJA
+MFEM_USE_MM
 
 The following options are CMake specific:
 
@@ -758,8 +728,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - LIBUNWIND
  - POSIXCLOCKS
  - PUMI
- - OCCA
- - RAJA
 
 The following built-in CMake packages are also used:
 

README

@@ -8,9 +8,9 @@
                                http://mfem.org
 
 MFEM is a modular parallel C++ library for finite element methods. Its goal is
-to enable high-performance scalable finite element discretization research and
-application development on a wide variety of platforms, ranging from laptops to
-supercomputers.
+to enable the research and development of scalable finite element discretization
+and solver algorithms through general finite element abstractions, accurate and
+flexible visualization, and tight integration with the hypre library.
 
 * For building instructions, see the file INSTALL, or type "make help".
 
@@ -39,24 +39,23 @@ conforming and non-conforming (AMR) adaptive refinement. Arbitrary element
 transformations, allowing for high-order mesh elements with curved boundaries,
 are also supported.
 
-When used as a "finite element to linear algebra translator", MFEM can take a
-problem described in terms of finite element-type objects, and produce the
-corresponding linear algebra vectors and fully or partially assembled operators,
-e.g. in the form of global sparse matrices or matrix-free operators. The library
-includes simple smoothers and Krylov solvers, such as PCG, MINRES and GMRES, as
-well as support for sequential sparse direct solvers from the SuiteSparse
+MFEM is commonly used as a "finite element to linear algebra translator", since
+it can take a problem described in terms of finite element-type objects, and
+produce the corresponding linear algebra vectors and sparse matrices. In order
+to facilitate this, MFEM uses compressed sparse row (CSR) sparse matrix storage
+and includes simple smoothers and Krylov solvers, such as PCG, MINRES and GMRES,
+as well as support for sequential sparse direct solvers from the SuiteSparse
 library. Nonlinear solvers (the Newton method), eigensolvers (LOBPCG), and
 several explicit and implicit Runge-Kutta time integrators are also available.
 
 MFEM supports MPI-based parallelism throughout the library, and can readily be
-used as a scalable unstructured finite element problem generator. As of version
-4.0, MFEM offers initial support for GPU acceleration, and programming models,
-such as CUDA, OCCA, RAJA and OpenMP. MFEM-based applications require minimal
-changes to switch from a serial to a high-performing MPI-parallel version of the
-code, where they can take advantage of the integrated linear solvers from the
-hypre library.  Comprehensive support for other external packages, e.g. PETSc
-and SUNDIALS is also included, giving access to many additional linear and
-nonlinear solvers, preconditioners, time integrators, etc.
+used as a scalable unstructured finite element problem generator. MFEM-based
+applications require minimal changes to transition from a serial to a
+high-performing parallel version of the code, where they can take advantage of
+the integrated scalable linear solvers from the hypre library. Comprehensive
+support for other external packages, e.g. PETSc and SUNDIALS is also included,
+giving access to many additional linear and nonlinear solvers, preconditioners,
+time integrators, etc.
 
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
 as the OpenGL visualization tool GLVis which is available at http://glvis.org.

config/XSDKDefaults.cmake

@@ -74,7 +74,7 @@
 
 IF (NOT COMMAND PRINT_VAR)
   FUNCTION(PRINT_VAR  VAR_NAME)
-    MESSAGE(STATUS "${VAR_NAME} = '${${VAR_NAME}}'")
+    MESSAGE("-- " "${VAR_NAME} = '${${VAR_NAME}}'")
   ENDFUNCTION()
 ENDIF()
 
@@ -166,14 +166,14 @@ IF (USE_XSDK_DEFAULTS)
     ENDIF()
     XSDK_HANDLE_LANG_DEFAULTS(Fortran  FC  "FFLAGS;FCFLAGS")
   ENDIF()
-
+  
   # Set XSDK defaults for other CMake variables
-
+  
   IF ("${BUILD_SHARED_LIBS}"  STREQUAL  "")
     MESSAGE("-- " "XSDK: Setting default BUILD_SHARED_LIBS=TRUE")
     SET(BUILD_SHARED_LIBS  TRUE  CACHE  BOOL  "Set by default in XSDK mode")
   ENDIF()
-
+  
   IF ("${CMAKE_BUILD_TYPE}"  STREQUAL  "")
     MESSAGE("-- " "XSDK: Setting default CMAKE_BUILD_TYPE=DEBUG")
     SET(CMAKE_BUILD_TYPE  DEBUG  CACHE  STRING  "Set by default in XSDK mode")
@@ -181,13 +181,6 @@ IF (USE_XSDK_DEFAULTS)
 
 ENDIF()
 
-
-##################################################################################
-#
-#               MFEM-specific additions: set TPL MFEM_USE_* defaults
-#
-##################################################################################
-
 IF (DEFINED TPL_ENABLE_MPI)
   SET(MFEM_USE_MPI ${TPL_ENABLE_MPI} CACHE BOOL "Enable MPI parallel build" FORCE)
 ENDIF()
@@ -259,15 +252,3 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_PUMI)
   SET(MFEM_USE_PUMI ${TPL_ENABLE_PUMI} CACHE BOOL "Enable PUMI" FORCE)
 ENDIF()
-
-IF (DEFINED TPL_ENABLE_CUDA)
-  SET(MFEM_USE_CUDA ${TPL_ENABLE_CUDA} CACHE BOOL "Enable CUDA" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_OCCA)
-  SET(MFEM_USE_OCCA ${TPL_ENABLE_OCCA} CACHE BOOL "Enable OCCA" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_RAJA)
-  SET(MFEM_USE_RAJA ${TPL_ENABLE_RAJA} CACHE BOOL "Enable RAJA" FORCE)
-ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -41,9 +41,6 @@ set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
-set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
-set(MFEM_USE_OCCA @MFEM_USE_OCCA@)
-set(MFEM_USE_RAJA @MFEM_USE_RAJA@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -30,12 +30,6 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)
 
-// MFEM source directory.
-#define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
-
-// MFEM install directory.
-#define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
-
 // Description of the git commit used to build MFEM.
 #cmakedefine MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
@@ -110,16 +104,6 @@
 // Enable MFEM functionality based on the PUMI library
 #cmakedefine MFEM_USE_PUMI
 
-// Build the GPU/CUDA-enabled version of the MFEM library.
-// Requires a CUDA compiler (nvcc).
-#cmakedefine MFEM_USE_CUDA
-
-// Enable MFEM functionality based on the RAJA library
-#cmakedefine MFEM_USE_RAJA
-
-// Enable MFEM functionality based on the OCCA library
-#cmakedefine MFEM_USE_OCCA
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.

config/cmake/modules/FindOCCA.cmake

@@ -1,19 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Defines the following variables:
-#   - OCCA_FOUND
-#   - OCCA_LIBRARIES
-#   - OCCA_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(OCCA OCCA OCCA_DIR "include" "occa.hpp" "lib" "occa"
-  "Paths to headers required by OCCA." "Libraries required by OCCA.")

config/cmake/modules/FindRAJA.cmake

@@ -1,30 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Defines the following variables:
-#   - RAJA_FOUND
-#   - RAJA_LIBRARIES
-#   - RAJA_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(RAJA RAJA RAJA_DIR "include" "RAJA/RAJA.hpp" "lib" "RAJA"
-  "Paths to headers required by RAJA." "Libraries required by RAJA.")
-
-if (NOT RAJA_CONFIG_CMAKE)
-   set(RAJA_CONFIG_CMAKE "${RAJA_DIR}/share/raja/cmake/raja-config.cmake")
-endif()
-if (EXISTS "${RAJA_CONFIG_CMAKE}")
-   include("${RAJA_CONFIG_CMAKE}")
-   if (ENABLE_CUDA AND NOT MFEM_USE_CUDA)
-      message(FATAL_ERROR
-         "RAJA is built with CUDA: MFEM_USE_CUDA=YES is required")
-   endif()
-endif()

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -232,12 +232,10 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
   # If we have the TPL_ versions of _INCLUDE_DIRS and _LIBRARIES then set the
   # standard ${Prefix} versions
   if (TPL_${Prefix}_INCLUDE_DIRS)
-    set(${Prefix}_INCLUDE_DIRS ${TPL_${Prefix}_INCLUDE_DIRS} CACHE STRING
-      "TPL_${Prefix}_INCLUDE_DIRS was found." FORCE)
+    set(${Prefix}_INCLUDE_DIRS ${TPL_${Prefix}_INCLUDE_DIRS} CACHE STRING "TPL_${Prefix}_INCLUDE_DIRS was found." FORCE)
   endif()
   if (TPL_${Prefix}_LIBRARIES)
-    set(${Prefix}_LIBRARIES ${TPL_${Prefix}_LIBRARIES} CACHE STRING
-      "TPL_${Prefix}_LIBRARIES was found." FORCE)
+    set(${Prefix}_LIBRARIES ${TPL_${Prefix}_LIBRARIES} CACHE STRING "TPL_${Prefix}_LIBRARIES was found." FORCE)
   endif()
 
   # Quick return
@@ -720,7 +718,7 @@ function(mfem_export_mk_files)
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GECKO MFEM_USE_GNUTLS
       MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA)
+      MFEM_USE_CONDUIT MFEM_USE_PUMI)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -728,7 +726,6 @@ function(mfem_export_mk_files)
       set(${var} NO)
     endif()
   endforeach()
-  # TODO: Add support for MFEM_USE_CUDA=YES
   set(MFEM_CXX ${CMAKE_CXX_COMPILER})
   set(MFEM_CPPFLAGS "")
   string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"

config/config.hpp

@@ -10,15 +10,18 @@
 // Software Foundation) version 2.1 dated February 1999.
 
 
-// Support out-of-source builds: if MFEM_CONFIG_FILE is defined, include it.
+// Support out-of-source builds: if MFEM_BUILD_DIR is defined, load the config
+// file MFEM_BUILD_DIR/config/_config.hpp.
 //
 // Otherwise, use the local file: _config.hpp.
 
 #ifndef MFEM_CONFIG_HPP
 #define MFEM_CONFIG_HPP
 
-#ifdef MFEM_CONFIG_FILE
-#include MFEM_CONFIG_FILE
+#ifdef MFEM_BUILD_DIR
+#define MFEM_QUOTE(a) #a
+#define MFEM_MAKE_PATH(x,y) MFEM_QUOTE(x/y)
+#include MFEM_MAKE_PATH(MFEM_BUILD_DIR,config/_config.hpp)
 #else
 #include "_config.hpp"
 #endif

config/config.hpp.in

@@ -121,20 +121,19 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI
 
-// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
+// Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 // #define MFEM_USE_CUDA
 
-// Build the AMD GPU/HIP-enabled version of the MFEM library.
-// Requires a HIP compiler (hipcc).
-// #define MFEM_USE_HIP
-
 // Enable functionality based on the RAJA library.
 // #define MFEM_USE_RAJA
 
 // Enable functionality based on the OCCA library.
 // #define MFEM_USE_OCCA
 
+// Enable MFEM's internal Memory Manager (needed e.g. for MFEM_USE_CUDA)
+// #define MFEM_USE_MM
+
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 

config/config.mk.in

@@ -42,9 +42,9 @@ MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
-MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
+MFEM_USE_MM            = @MFEM_USE_MM@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -42,9 +42,6 @@ option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
-option(MFEM_USE_CUDA "Enable CUDA" OFF)
-option(MFEM_USE_OCCA "Enable OCCA" OFF)
-option(MFEM_USE_RAJA "Enable RAJA" OFF)
 
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 
@@ -62,16 +59,13 @@ option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 # set(CXX g++)
 # set(MPICXX mpicxx)
 
-# Set the target CUDA architecture
-set(CUDA_ARCH "sm_60" CACHE STRING "Target CUDA architecture.")
-
 set(MFEM_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 # The *_DIR paths below will be the first place searched for the corresponding
 # headers and library. If these fail, then standard cmake search is performed.
 # Note: if the variables are already in the cache, they are not overwritten.
 
-set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
+set(HYPRE_DIR "${MFEM_DIR}/../hypre-2.10.0b/src/hypre" CACHE PATH
     "Path to the hypre library.")
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
@@ -160,9 +154,6 @@ set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
     "Directory where PUMI is installed")
 
-set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
-set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")

config/defaults.mk

@@ -46,14 +46,6 @@ CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
 CUDA_XCOMPILER = -Xcompiler=
 CUDA_XLINKER   = -Xlinker=
 
-# HIP configuration options
-HIP_CXX = hipcc
-# The HIP_ARCH option specifies the AMD GPU processor, similar to CUDA_ARCH. For
-# example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo),
-# gfx900, gfx1010, etc.
-HIP_ARCH = gfx900
-HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
-
 ifneq ($(NOTMAC),)
    AR      = ar
    ARFLAGS = cruv
@@ -103,9 +95,9 @@ MFEM_MPI_NP = 4
 # config.hpp. The values below are the defaults for generating the actual values
 # in config.mk and config.hpp.
 
-MFEM_USE_MPI           = YES
+MFEM_USE_MPI           = NO
 MFEM_USE_METIS         = $(MFEM_USE_MPI)
-MFEM_USE_METIS_5       = YES
+MFEM_USE_METIS_5       = NO
 MFEM_DEBUG             = NO
 MFEM_USE_EXCEPTIONS    = NO
 MFEM_USE_GZSTREAM      = NO
@@ -130,9 +122,9 @@ MFEM_USE_SIDRE         = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_CUDA          = NO
-MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
+MFEM_USE_MM            = NO
 
 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -144,7 +136,7 @@ LIBUNWIND_OPT = -g
 LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 
 # HYPRE library configuration (needed to build the parallel version)
-HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
+HYPRE_DIR = @MFEM_DIR@/../hypre-2.10.0b/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
 
@@ -308,21 +300,19 @@ PUMI_OPT = -I$(PUMI_DIR)/include
 PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
    -llion -lmth -lapf_zoltan -lspr
 
-# CUDA library configuration (currently not needed)
+# CUDA library configuration. Since we compile and link with nvcc (when CUDA is
+# enabled) we only need to explicitly link with the CUDA driver, libcuda.*,
+# which is usually in a system path.
 CUDA_OPT =
-CUDA_LIB =
-
-# HIP library configuration (currently not needed)
-HIP_OPT =
-HIP_LIB =
+CUDA_LIB = $(if $(NOTMAC),,-L/usr/local/cuda/lib) -lcuda
 
 # OCCA library configuration
-OCCA_DIR = @MFEM_DIR@/../occa
+OCCA_DIR ?= @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 
 # RAJA library configuration
-RAJA_DIR = @MFEM_DIR@/../raja
+RAJA_DIR ?= @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
    RAJA_OPT += -I$(CUB_DIR)

config/makefile

@@ -36,7 +36,6 @@ CONFIG_MK = config.mk
 all: header config-mk
 
 MPI = $(MFEM_USE_MPI:NO=)
-GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
 GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
@@ -45,7 +44,7 @@ SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
-	$(GHV_CXX) ${GHV_FLAGS} $(SRC)$(GHV).cpp -o $(GHV)
+	$(MFEM_CXX) ${GHV_FLAGS} $(SRC)$(GHV).cpp -o $(GHV)
 $(GHV).out: $(GHV)
 	./$(GHV) > $(GHV).out
 .INTERMEDIATE: $(GHV) $(GHV).out

config/sample-runs.sh

@@ -18,8 +18,6 @@ run_prefix=""
 run_vg="valgrind --leak-check=full --show-reachable=yes --track-origins=yes"
 run_suffix="-no-vis"
 skip_gen_meshes="yes"
-# filter-out device runs ("no") or non-device runs ("yes"):
-device_runs="no"
 cur_dir="${PWD}"
 mfem_dir="$(cd "$(dirname "$0")"/.. && pwd)"
 mfem_build_dir=""
@@ -150,11 +148,6 @@ function extract_sample_runs()
    if [ "$skip_gen_meshes" == "yes" ]; then
       runs=`printf "%s" "$runs" | grep -v ".* -m .*\.gen"`
    fi
-   if [ "$device_runs" == "yes" ]; then
-      runs=`printf "%s" "$runs" | grep ".* -d .*"`
-   else
-      runs=`printf "%s" "$runs" | grep -v ".* -d .*"`
-   fi
    IFS=$'\n'
    runs=(${runs})
    IFS="${old_IFS}"
@@ -176,9 +169,6 @@ function help_message()
       -g <dir> <pattern>
                   Specify explicitly a group (dir + file pattern) to run; This
                   option can be used multiple times to define multiple groups
-      -dev        configure only sample runs using devices.
-                  To test with a parallel build, the parallel (-p|-par) option
-                  should be set first on the command line.
       -v          Enable valgrind
       -o <dir>    [${output_dir:-"<empty>: output goes to stdout"}]
                   If not empty, save output to files inside <dir>
@@ -263,7 +253,7 @@ case "$1" in
    -h|-help)
       opt_help="yes"
       ;;
-   -p|-par)
+   -p|-parallel)
       mfem_config="MFEM_USE_MPI=YES MFEM_DEBUG=NO"
       ;;
    -g)
@@ -274,10 +264,6 @@ case "$1" in
       groups=("${groups[@]}" "${test_group}")
       shift 2
       ;;
-   -dev)
-       device_runs="yes"
-       mfem_config+=" MFEM_USE_CUDA=YES MFEM_USE_OCCA=YES MFEM_USE_RAJA=YES MFEM_USE_OPENMP=YES"
-      ;;
    -v)
       valgrind="yes"
       ;;
@@ -308,10 +294,6 @@ case "$1" in
    -n)
       run_prefix="echo"
       ;;
-   -*)
-      echo "unknown option: '$1'"
-      exit 1
-      ;;
    *=*)
       eval $1
       ;;

config/tconfig.hpp

@@ -43,14 +43,15 @@
 #define MFEM_ALIGN_SIZE(size,type) \
    MFEM_ROUNDUP(size,(MFEM_SIMD_SIZE)/sizeof(type))
 
-#ifdef MFEM_COUNT_FLOPS
 namespace mfem
 {
 namespace internal
 {
-extern long long flop_count;
+long long flop_count;
 }
 }
+
+#ifdef MFEM_COUNT_FLOPS
 #define MFEM_FLOPS_RESET() (mfem::internal::flop_count = 0)
 #define MFEM_FLOPS_ADD(cnt) (mfem::internal::flop_count += (cnt))
 #define MFEM_FLOPS_GET() (mfem::internal::flop_count)

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.0.1
+PROJECT_NUMBER         = v3.4.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/CodeDocumentation.dox

@@ -35,12 +35,6 @@ namespace mfem {
  * - HypreParMatrix and HypreParVector
  * - HypreSolver and other \link hypre.hpp hypre classes\endlink
  *
- * <H3>Main GPU classes</H3>
- * - Device
- * - Memory
- * - MemoryManager
- * - MFEM_FORALL macro in forall.hpp
- *
  * <H3>Example codes</H3>
  * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
  * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
@@ -79,8 +73,8 @@ namespace mfem {
  * - <a class="el" href="ex19p_8cpp_source.html">Example 19p</a>: parallel incompressible nonlinear elasticity
  * - <a class="el" href="ex20_8cpp_source.html">Example 20</a>: symplectic ODE integration
  * - <a class="el" href="ex20p_8cpp_source.html">Example 20p</a>: parallel symplectic ODE integration
- * - <a class="el" href="ex21_8cpp_source.html">Example 21</a>: adaptive mesh refinement for linear elasticity
- * - <a class="el" href="ex21p_8cpp_source.html">Example 21p</a>: parallel adaptive mesh refinement for linear elasticity
+ * - <a class="el" href="ex22_8cpp_source.html">Example 22</a>: adaptive mesh refinement for linear elasticity
+ * - <a class="el" href="ex22p_8cpp_source.html">Example 22p</a>: parallel adaptive mesh refinement for linear elasticity
  *
  * <H4>SUNDIALS Examples</H4>
  * - Variants of Examples

examples/CMakeLists.txt

@@ -28,7 +28,7 @@ list(APPEND ALL_EXE_SRCS
   ex19.cpp
   ex20.cpp
   ex21.cpp
-  ex23.cpp
+  ex22.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -54,7 +54,7 @@ if (MFEM_USE_MPI)
     ex19p.cpp
     ex20p.cpp
     ex21p.cpp
-    ex23p.cpp
+    ex22p.cpp
     )
 endif()
 

examples/ex1.cpp

@@ -26,12 +26,12 @@
 //               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
 //
 // Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
+//             > ex1 -pa -d cuda
+//             > ex1 -pa -d raja-cuda
+//             > ex1 -pa -d occa-cuda
+//             > ex1 -pa -d raja-omp
+//             > ex1 -pa -d occa-omp
+//             > ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -62,7 +62,7 @@ int main(int argc, char *argv[])
    int order = 1;
    bool static_cond = false;
    bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
    bool visualization = true;
 
    OptionsParser args(argc, argv);
@@ -75,7 +75,7 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
@@ -88,18 +88,13 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
    //    largest number that gives a final mesh with no more than 50,000
    //    elements.
@@ -112,7 +107,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
    //    Lagrange finite elements of the specified order. If order < 1, we
    //    instead use an isoparametric/isogeometric space.
    FiniteElementCollection *fec;
@@ -133,7 +128,7 @@ int main(int argc, char *argv[])
    cout << "Number of finite element unknowns: "
         << fespace->GetTrueVSize() << endl;
 
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined by marking all
    //    the boundary attributes from the mesh as essential (Dirichlet) and
    //    converting them to a list of true dofs.
@@ -145,7 +140,7 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
    //    the basis functions in the finite element fespace.
    LinearForm *b = new LinearForm(fespace);
@@ -153,6 +148,12 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(one));
    b->Assemble();
 
+   // 7. Set device config parameters from the command line options and switch
+   //    to working on the device.
+   Device::Configure(device);
+   Device::Print();
+   Device::Enable();
+
    // 8. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero,
    //    which satisfies the boundary conditions.
@@ -202,7 +203,10 @@ int main(int argc, char *argv[])
    // 12. Recover the solution as a finite element grid function.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 13. Save the refined mesh and the solution. This output can be viewed later
+   // 13. Switch back to the host.
+   Device::Disable();
+
+   // 14. Save the refined mesh and the solution. This output can be viewed later
    //     using GLVis: "glvis -m refined.mesh -g sol.gf".
    ofstream mesh_ofs("refined.mesh");
    mesh_ofs.precision(8);
@@ -211,7 +215,7 @@ int main(int argc, char *argv[])
    sol_ofs.precision(8);
    x.Save(sol_ofs);
 
-   // 14. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -221,7 +225,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *mesh << x << flush;
    }
 
-   // 15. Free the used memory.
+   // 16. Free the used memory.
    delete a;
    delete b;
    delete fespace;

examples/ex19.cpp

@@ -144,7 +144,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options
-   const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/beam-hex.mesh";
    int ref_levels = 0;
    int order = 2;
    bool visualization = true;

examples/ex19p.cpp

@@ -150,7 +150,7 @@ int main(int argc, char *argv[])
    MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 
    // 2. Parse command-line options
-   const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/beam-hex.mesh";
    int ser_ref_levels = 0;
    int par_ref_levels = 0;
    int order = 2;

examples/ex1p.cpp

@@ -26,9 +26,9 @@
 //               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
 //
 // Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
+//             > mpirun -np 4 ex1p -pa -d cuda
+//             > mpirun -np 4 ex1p -pa -d occa-cuda
+//             > mpirun -np 4 ex1p -pa -d raja-omp
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -65,7 +65,7 @@ int main(int argc, char *argv[])
    int order = 1;
    bool static_cond = false;
    bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
    bool visualization = true;
 
    OptionsParser args(argc, argv);
@@ -78,7 +78,7 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
@@ -98,18 +98,13 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 10,000 elements.
@@ -122,7 +117,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -135,7 +130,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange finite elements of the specified order. If
    //    order < 1, we instead use an isoparametric/isogeometric space.
    FiniteElementCollection *fec;
@@ -162,7 +157,7 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined
    //    by marking all the boundary attributes from the mesh as essential
    //    (Dirichlet) and converting them to a list of true dofs.
@@ -174,7 +169,7 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system, which in this case is
    //    (1,phi_i) where phi_i are the basis functions in fespace.
    ParLinearForm *b = new ParLinearForm(fespace);
@@ -182,6 +177,12 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(one));
    b->Assemble();
 
+   // 9. Set device config parameters from the command line options and switch
+   //    to working on the device.
+   Device::Configure(device);
+   if (myid == 0) { Device::Print(); }
+   Device::Enable();
+
    // 10. Define the solution vector x as a parallel finite element grid function
    //     corresponding to fespace. Initialize x with initial guess of zero,
    //     which satisfies the boundary conditions.
@@ -224,7 +225,10 @@ int main(int argc, char *argv[])
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 15. Save the refined mesh and the solution in parallel. This output can
+   // 15. Switch back to the host.
+   Device::Disable();
+
+   // 16. Save the refined mesh and the solution in parallel. This output can
    //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_name;
@@ -240,7 +244,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -251,7 +255,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 17. Free the used memory.
+   // 18. Free the used memory.
    delete a;
    delete b;
    delete fespace;

examples/ex21.cpp

@@ -1,34 +1,40 @@
-//                                MFEM Example 21
+//                               MFEM Example 21
 //
 // Compile with: make ex21
 //
-// Sample runs:  ex21
-//               ex21 -o 3
-//               ex21 -m ../data/beam-quad.mesh
-//               ex21 -m ../data/beam-quad.mesh -o 3
-//               ex21 -m ../data/beam-quad.mesh -o 3 -f 1
-//               ex21 -m ../data/beam-tet.mesh
-//               ex21 -m ../data/beam-tet.mesh -o 2
-//               ex21 -m ../data/beam-hex.mesh
-//               ex21 -m ../data/beam-hex.mesh -o 2
+// Sample runs:  ex21 -m ../data/inline-segment.mesh -o 3
+//               ex21 -m ../data/inline-tri.mesh -o 3
+//               ex21 -m ../data/inline-quad.mesh -o 3
+//               ex21 -m ../data/inline-quad.mesh -o 3 -p 1
+//               ex21 -m ../data/inline-quad.mesh -o 3 -p 2
+//               ex21 -m ../data/inline-tet.mesh -o 2
+//               ex21 -m ../data/inline-hex.mesh -o 2
+//               ex21 -m ../data/inline-hex.mesh -o 2 -p 1
+//               ex21 -m ../data/inline-hex.mesh -o 2 -p 2
+//               ex21 -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Description:  This is a version of Example 2 with a simple adaptive mesh
-//               refinement loop. The problem being solved is again the linear
-//               elasticity describing a multi-material cantilever beam.
-//               The problem is solved on a sequence of meshes which
-//               are locally refined in a conforming (triangles, tetrahedrons)
-//               or non-conforming (quadrilaterals, hexahedra) manner according
-//               to a simple ZZ error estimator.
+// Description:  This example code demonstrates the use of MFEM to define and
+//               solve simple complex-valued linear systems.  We implement three
+//               variants of a damped harmonic oscillator:
 //
-//               The example demonstrates MFEM's capability to work with both
-//               conforming and nonconforming refinements, in 2D and 3D, on
-//               linear and curved meshes. Interpolation of functions from
-//               coarse to fine meshes, as well as persistent GLVis
-//               visualization are also illustrated.
+//               1) A scalar H1 field
+//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
+//
+//               2) A vector H(Curl) field
+//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
+//
+//               3) A vector H(Div) field
+//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
+//
+//               In each case the field is driven by a forced oscillation, with
+//               angular frequency omega, imposed at the boundary or a portion
+//               of the boundary.
+//
+//               In electromagnetics the coefficients are typically named the
+//               permeability, mu = 1/a, permittivity, epsilon = b, and
+//               conductivity, sigma = c.  The user can specify these constants
+//               using either set of names.
 //
-//               We recommend viewing Examples 2 and 6 before viewing this
-//               example.
-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -36,24 +42,61 @@
 using namespace std;
 using namespace mfem;
 
+static double mu_ = 1.0;
+static double epsilon_ = 1.0;
+static double sigma_ = 20.0;
+static double omega_ = 10.0;
+
+double u0_real_exact(const Vector &);
+double u0_imag_exact(const Vector &);
+
+void u1_real_exact(const Vector &, Vector &);
+void u1_imag_exact(const Vector &, Vector &);
+
+void u2_real_exact(const Vector &, Vector &);
+void u2_imag_exact(const Vector &, Vector &);
+
+bool check_for_inline_mesh(const char * mesh_file);
+
 int main(int argc, char *argv[])
 {
    // 1. Parse command-line options.
-   const char *mesh_file = "../data/beam-tri.mesh";
+   const char *mesh_file = "../data/inline-quad.mesh";
+   int ref_levels = 0;
    int order = 1;
-   bool static_cond = false;
-   int flux_averaging = 0;
+   int prob = 0;
+   double freq = -1.0;
+   double a_coef = 0.0;
    bool visualization = 1;
+   bool herm_conv = true;
+   bool exact_sol = true;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&flux_averaging, "-f", "--flux-averaging",
-                  "Flux averaging: 0 - global, 1 - by mesh attribute.");
+   args.AddOption(&prob, "-p", "--problem-type",
+                  "Choose from 0: H_1, 1: H(Curl), or 2: H(Div) "
+                  "damped harmonic oscillator.");
+   args.AddOption(&a_coef, "-a", "--stiffness-coef",
+                  "Stiffness coefficient (spring constant or 1/mu).");
+   args.AddOption(&epsilon_, "-b", "--mass-coef",
+                  "Mass coefficient (or epsilon).");
+   args.AddOption(&sigma_, "-c", "--damping-coef",
+                  "Damping coefficient (or sigma).");
+   args.AddOption(&mu_, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon_, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma_, "-sigma", "--conductivity",
+                  "Conductivity (or damping constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -65,246 +108,370 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
+   if ( a_coef != 0.0 )
+   {
+      mu_ = 1.0 / a_coef;
+   }
+   if ( freq > 0.0 )
+   {
+      omega_ = 2.0 * M_PI * freq;
+   }
+
+   exact_sol = check_for_inline_mesh(mesh_file);
+   if (exact_sol)
+   {
+      cout << "Identified an 'inline' mesh" << endl;
+   }
+
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+
    // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
+   //    with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
 
-   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
+   // 3. Refine the mesh to increase resolution. In this example we do
+   //    'ref_levels' of uniform refinement where the user specifies
+   //    the number of levels with the '-r' option.
+   for (int l = 0; l < ref_levels; l++)
    {
-      cerr << "\nInput mesh should have at least two materials and "
-           << "two boundary attributes! (See schematic in ex2.cpp)\n"
-           << endl;
-      return 3;
+      mesh->UniformRefinement();
    }
 
-   // 3. Since a NURBS mesh can currently only be refined uniformly, we need to
-   //    convert it to a piecewise-polynomial curved mesh. First we refine the
-   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext)
+   // 4. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
+   //    order.
+   if (dim == 1 && prob != 0 )
    {
-      for (int i = 0; i < 2; i++)
+      cout << "Switching to problem type 0, H1 basis functions, "
+           << "for 1 dimensional mesh." << endl;
+      prob = 0;
+   }
+
+   FiniteElementCollection *fec;
+   switch (prob)
+   {
+      case 0:  fec = new H1_FECollection(order, dim);      break;
+      case 1:  fec = new ND_FECollection(order, dim);      break;
+      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
+        << endl;
+
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined based on the type
+   //    of mesh and the problem type.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      if (exact_sol)
       {
-         mesh.UniformRefinement();
+         switch (prob)
+         {
+            case 0:   ess_bdr = 0; ess_bdr[0] = 1;  break;
+            default:  ess_bdr = 1; ess_bdr[2] = 0;  break;
+         }
       }
-      mesh.SetCurvature(2);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 4. Define a finite element space on the mesh. The polynomial order is
-   //    one (linear) by default, but this can be changed on the command line.
-   H1_FECollection fec(order, dim);
-   FiniteElementSpace fespace(&mesh, &fec, dim);
+   // 6. Set up the linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system.
+   ComplexLinearForm b(fespace, conv);
+   b.Vector::operator=(0.0);
 
-   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
-   //    the right-hand side of the FEM linear system. In this case, b_i equals
-   //    the boundary integral of f*phi_i where f represents a "pull down"
-   //    force on the Neumann part of the boundary and phi_i are the basis
-   //    functions in the finite element fespace. The force is defined by the
-   //    VectorArrayCoefficient object f, which is a vector of Coefficient
-   //    objects. The fact that f is non-zero on boundary attribute 2 is
-   //    indicated by the use of piece-wise constants coefficient for its last
-   //    component. We don't assemble the discrete problem yet, this will be
-   //    done in the main loop.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
+   // 7. Define the solution vector u as a finite element grid function
+   //    corresponding to fespace. Initialize u with initial guess of 1+0i
+   //    or the exact solution if it is known.
+   ComplexGridFunction u(fespace);
+   ComplexGridFunction * u_exact = NULL;
+   if (exact_sol) { u_exact = new ComplexGridFunction(fespace); }
+
+   FunctionCoefficient u0_r(u0_real_exact);
+   FunctionCoefficient u0_i(u0_imag_exact);
+   VectorFunctionCoefficient u1_r(dim, u1_real_exact);
+   VectorFunctionCoefficient u1_i(dim, u1_imag_exact);
+   VectorFunctionCoefficient u2_r(dim, u2_real_exact);
+   VectorFunctionCoefficient u2_i(dim, u2_imag_exact);
+
+   ConstantCoefficient zeroCoef(0.0);
+   ConstantCoefficient oneCoef(1.0);
+
+   Vector zeroVec(dim); zeroVec = 0.0;
+   Vector  oneVec(dim);  oneVec = 0.0; oneVec[(prob==2)?(dim-1):0] = 1.0;
+   VectorConstantCoefficient zeroVecCoef(zeroVec);
+   VectorConstantCoefficient oneVecCoef(oneVec);
+
+   switch (prob)
    {
-      f.Set(i, new ConstantCoefficient(0.0));
+      case 0:
+         u.ProjectBdrCoefficient(oneCoef, zeroCoef, ess_bdr);
+         if (exact_sol) { u_exact->ProjectCoefficient(u0_r, u0_i); }
+         break;
+      case 1:
+         u.ProjectBdrCoefficientTangent(oneVecCoef, zeroVecCoef, ess_bdr);
+         if (exact_sol) { u_exact->ProjectCoefficient(u1_r, u1_i); }
+         break;
+      case 2:
+         u.ProjectBdrCoefficientNormal(oneVecCoef, zeroVecCoef, ess_bdr);
+         if (exact_sol) { u_exact->ProjectCoefficient(u2_r, u2_i); }
+         break;
    }
+
+   if (visualization && exact_sol)
    {
-      Vector pull_force(mesh.bdr_attributes.Max());
-      pull_force = 0.0;
-      pull_force(1) = -1.0e-2;
-      f.Set(dim-1, new PWConstCoefficient(pull_force));
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
+      sol_sock_r << "solution\n" << *mesh << u_exact->real()
+                 << "window_title 'Exact Real Part'" << flush;
+      sol_sock_i << "solution\n" << *mesh << u_exact->imag()
+                 << "window_title 'Exact Imaginary Part'" << flush;
    }
 
-   LinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+   // 8. Set up the sesquilinear form a(.,.) on the finite element
+   //    space corresponding to the damped harmonic oscillator operator
+   //    of the appropriate type:
+   //
+   //    0) A scalar H1 field
+   //       -Div(a Grad) - omega^2 b + i omega c
+   //
+   //    1) A vector H(Curl) field
+   //       Curl(a Curl) - omega^2 b + i omega c
+   //
+   //    2) A vector H(Div) field
+   //       -Grad(a Div) - omega^2 b + i omega c
+   //
+   ConstantCoefficient stiffnessCoef(1.0/mu_);
+   ConstantCoefficient massCoef(-omega_ * omega_ * epsilon_);
+   ConstantCoefficient lossCoef(omega_ * sigma_);
+   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
 
-   // 6. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the linear elasticity integrator with piece-wise
-   //    constants coefficient lambda and mu.
-   Vector lambda(mesh.attributes.Max());
-   lambda = 1.0;
-   lambda(0) = lambda(1)*50;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(mesh.attributes.Max());
-   mu = 1.0;
-   mu(0) = mu(1)*50;
-   PWConstCoefficient mu_func(mu);
+   SesquilinearForm *a = new SesquilinearForm(fespace, conv);
+   switch (prob)
+   {
+      case 0:
+         a->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
+                                NULL);
+         a->AddDomainIntegrator(new MassIntegrator(massCoef),
+                                new MassIntegrator(lossCoef));
+         break;
+      case 1:
+         a->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
+                                NULL);
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                                new VectorFEMassIntegrator(lossCoef));
+         break;
+      case 2:
+         a->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
+                                NULL);
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                                new VectorFEMassIntegrator(lossCoef));
+         break;
+   }
 
-   BilinearForm a(&fespace);
-   BilinearFormIntegrator *integ =
-      new ElasticityIntegrator(lambda_func,mu_func);
-   a.AddDomainIntegrator(integ);
-   if (static_cond) { a.EnableStaticCondensation(); }
+   // 9. Assemble the bilinear form and the corresponding linear
+   //    system, applying any necessary transformations such as:
+   //    assembly, eliminating boundary conditions, applying conforming
+   //    constraints for non-conforming AMR, etc.
+   a->Assemble();
 
-   // 7. The solution vector x and the associated finite element grid function
-   //    will be maintained over the AMR iterations. We initialize it to zero.
-   Vector zero_vec(dim);
-   zero_vec = 0.0;
-   VectorConstantCoefficient zero_vec_coeff(zero_vec);
-   GridFunction x(&fespace);
-   x = 0.0;
+   OperatorHandle A;
+   Vector B, U;
 
-   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking only
-   //    boundary attribute 1 from the mesh as essential and converting it to a
-   //    list of true dofs.  The conversion to true dofs will be done in the
-   //    main loop.
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   ess_bdr = 0;
-   ess_bdr[0] = 1;
+   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;
 
-   // 9. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
+   {
+      ComplexSparseMatrix * Asp =
+         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
+
+      cout << "Size of linear system: "
+           << 2 * Asp->real().Width() << endl << endl;
+   }
+
+   // 10. Define and apply a GMRES solver for AU=B.
+   {
+      GMRESSolver gmres;
+      gmres.SetOperator(*A.Ptr());
+      gmres.SetRelTol(1e-12);
+      gmres.SetMaxIter(1000);
+      gmres.SetPrintLevel(1);
+      gmres.Mult(B, U);
+   }
+
+   // 11. Recover the solution as a finite element grid function and
+   //     compute the errors if the exact solution is known.
+   a->RecoverFEMSolution(U, b, u);
+
+   if (exact_sol)
+   {
+      double err_r = -1.0;
+      double err_i = -1.0;
+
+      switch (prob)
+      {
+         case 0:
+            err_r = u.real().ComputeL2Error(u0_r);
+            err_i = u.imag().ComputeL2Error(u0_i);
+            break;
+         case 1:
+            err_r = u.real().ComputeL2Error(u1_r);
+            err_i = u.imag().ComputeL2Error(u1_i);
+            break;
+         case 2:
+            err_r = u.real().ComputeL2Error(u2_r);
+            err_i = u.imag().ComputeL2Error(u2_i);
+            break;
+      }
+
+      cout << endl;
+      cout << "|| Re (u_h - u) ||_{L^2} = " << err_r << endl;
+      cout << "|| Im (u_h - u) ||_{L^2} = " << err_i << endl;
+      cout << endl;
+   }
+
+   // 12. Save the refined mesh and the solution. This output can be
+   //     viewed later using GLVis: "glvis -m mesh -g sol".
+   {
+      ofstream mesh_ofs("refined.mesh");
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+
+      ofstream sol_r_ofs("sol_r.gf");
+      ofstream sol_i_ofs("sol_i.gf");
+      sol_r_ofs.precision(8);
+      sol_i_ofs.precision(8);
+      u.real().Save(sol_r_ofs);
+      u.imag().Save(sol_i_ofs);
+   }
+
+   // 13. Send the solution by socket to a GLVis server.
    if (visualization)
    {
-      sol_sock.open(vishost, visport);
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
+      sol_sock_r << "solution\n" << *mesh << u.real()
+                 << "window_title 'Comp Real Part'" << flush;
+      sol_sock_i << "solution\n" << *mesh << u.imag()
+                 << "window_title 'Comp Imaginary Part'" << flush;
+   }
+   if (visualization && exact_sol)
+   {
+      *u_exact -= u;
+
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
+      sol_sock_r << "solution\n" << *mesh << u_exact->real()
+                 << "window_title 'Exact-Comp Real Part'" << flush;
+      sol_sock_i << "solution\n" << *mesh << u_exact->imag()
+                 << "window_title 'Exact-Comp Imaginary Part'" << flush;
+   }
+   if (visualization)
+   {
+      GridFunction u_t(fespace);
+      u_t = u.real();
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
       sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << u_t
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "pause\n" << flush;
+
+      cout << "GLVis visualization paused."
+           << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 32;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos( 2.0 * M_PI * t), u.real(),
+             sin(-2.0 * M_PI * t), u.imag(), u_t);
+         sol_sock << "solution\n" << *mesh << u_t
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
    }
 
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
-   //     recover a smoothed flux (stress) that is subtracted from the element
-   //     flux to get an error indicator. We need to supply the space for the
-   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
-   //     Here, tdim represents the number of components for a symmetric (dim x
-   //     dim) tensor.
-   const int tdim = dim*(dim+1)/2;
-   FiniteElementSpace flux_fespace(&mesh, &fec, tdim);
-   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
-   estimator.SetFluxAveraging(flux_averaging);
-
-   // 11. A refiner selects and refines elements based on a refinement strategy.
-   //     The strategy here is to refine elements with errors larger than a
-   //     fraction of the maximum element error. Other strategies are possible.
-   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
-   refiner.SetTotalErrorFraction(0.7);
-
-   // 12. The main AMR loop. In each iteration we solve the problem on the
-   //     current mesh, visualize the solution, and refine the mesh.
-   const int max_dofs = 50000;
-   const int max_amr_itr = 20;
-   for (int it = 0; it <= max_amr_itr; it++)
-   {
-      int cdofs = fespace.GetTrueVSize();
-      cout << "\nAMR iteration " << it << endl;
-      cout << "Number of unknowns: " << cdofs << endl;
-
-      // 13. Assemble the stiffness matrix and the right-hand side.
-      a.Assemble();
-      b.Assemble();
-
-      // 14. Set Dirichlet boundary values in the GridFunction x.
-      //     Determine the list of Dirichlet true DOFs in the linear system.
-      Array<int> ess_tdof_list;
-      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-      // 15. Create the linear system: eliminate boundary conditions, constrain
-      //     hanging nodes and possibly apply other transformations. The system
-      //     will be solved for true (unconstrained) DOFs only.
-      SparseMatrix A;
-      Vector B, X;
-      const int copy_interior = 1;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-
-#ifndef MFEM_USE_SUITESPARSE
-      // 16. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-      //     solve the linear system with PCG.
-      GSSmoother M(A);
-      PCG(A, M, B, X, 3, 2000, 1e-12, 0.0);
-#else
-      // 16. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
-      //     the linear system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(A);
-      umf_solver.Mult(B, X);
-#endif
-
-      // 17. After solving the linear system, reconstruct the solution as a
-      //     finite element GridFunction. Constrained nodes are interpolated
-      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
-      a.RecoverFEMSolution(X, b, x);
-
-      // 18. Send solution by socket to the GLVis server.
-      if (visualization && sol_sock.good())
-      {
-         GridFunction nodes(&fespace), *nodes_p = &nodes;
-         mesh.GetNodes(nodes);
-         nodes += x;
-         int own_nodes = 0;
-         mesh.SwapNodes(nodes_p, own_nodes);
-         x.Neg(); // visualize the backward displacement
-         sol_sock << "solution\n" << mesh << x << flush;
-         x.Neg();
-         mesh.SwapNodes(nodes_p, own_nodes);
-         if (it == 0)
-         {
-            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
-         }
-         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
-                  << "pause" << endl;
-         cout << "Visualization paused. "
-              "Press <space> in the GLVis window to continue." << endl;
-      }
-
-      if (cdofs > max_dofs)
-      {
-         cout << "Reached the maximum number of dofs. Stop." << endl;
-         break;
-      }
-
-      // 19. Call the refiner to modify the mesh. The refiner calls the error
-      //     estimator to obtain element errors, then it selects elements to be
-      //     refined and finally it modifies the mesh. The Stop() method can be
-      //     used to determine if a stopping criterion was met.
-      refiner.Apply(mesh);
-      if (refiner.Stop())
-      {
-         cout << "Stopping criterion satisfied. Stop." << endl;
-         break;
-      }
-
-      // 20. Update the space to reflect the new state of the mesh. Also,
-      //     interpolate the solution x so that it lies in the new space but
-      //     represents the same function. This saves solver iterations later
-      //     since we'll have a good initial guess of x in the next step.
-      //     Internally, FiniteElementSpace::Update() calculates an
-      //     interpolation matrix which is then used by GridFunction::Update().
-      fespace.Update();
-      x.Update();
-
-      // 21. Inform also the bilinear and linear forms that the space has
-      //     changed.
-      a.Update();
-      b.Update();
-   }
-
-   {
-      ofstream mesh_ref_out("ex21_reference.mesh");
-      mesh_ref_out.precision(16);
-      mesh.Print(mesh_ref_out);
-
-      ofstream mesh_out("ex21_deformed.mesh");
-      mesh_out.precision(16);
-      GridFunction nodes(&fespace), *nodes_p = &nodes;
-      mesh.GetNodes(nodes);
-      nodes += x;
-      int own_nodes = 0;
-      mesh.SwapNodes(nodes_p, own_nodes);
-      mesh.Print(mesh_out);
-      mesh.SwapNodes(nodes_p, own_nodes);
-
-      ofstream x_out("ex21_displacement.sol");
-      x_out.precision(16);
-      x.Save(x_out);
-   }
+   // 14. Free the used memory.
+   delete a;
+   delete u_exact;
+   delete fespace;
+   delete fec;
+   delete mesh;
 
    return 0;
 }
+
+bool check_for_inline_mesh(const char * mesh_file)
+{
+   string file(mesh_file);
+   size_t p0 = file.find_last_of("/");
+   string s0 = file.substr((p0==string::npos)?0:(p0+1),7);
+   return s0 == "inline-";
+}
+
+complex<double> u0_exact(const Vector &x)
+{
+   int dim = x.Size();
+   complex<double> i(0.0, 1.0);
+   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
+   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
+   return std::exp(-i * kappa * x[dim - 1]);
+}
+
+double u0_real_exact(const Vector &x)
+{
+   return u0_exact(x).real();
+}
+
+double u0_imag_exact(const Vector &x)
+{
+   return u0_exact(x).imag();
+}
+
+void u1_real_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[0] = u0_real_exact(x);
+}
+
+void u1_imag_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[0] = u0_imag_exact(x);
+}
+
+void u2_real_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[dim-1] = u0_real_exact(x);
+}
+
+void u2_imag_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[dim-1] = u0_imag_exact(x);
+}

examples/ex21p.cpp

@@ -1,65 +1,122 @@
-//                                MFEM Example 21
+//                       MFEM Example 21 - Parallel Version
 //
 // Compile with: make ex21p
 //
-// Sample runs:  mpirun -np 4 ex21p
-//               mpirun -np 4 ex21p -o 3
-//               mpirun -np 4 ex21p -m ../data/beam-quad.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-quad.mesh -o 3
-//               mpirun -np 4 ex21p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-tet.mesh -o 2
-//               mpirun -np 4 ex21p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-hex.mesh -o 2
+// Sample runs:  mpirun -np 4 ex21p -m ../data/inline-segment.mesh -o 3
+//               mpirun -np 4 ex21p -m ../data/inline-tri.mesh -o 3
+//               mpirun -np 4 ex21p -m ../data/inline-quad.mesh -o 3
+//               mpirun -np 4 ex21p -m ../data/inline-quad.mesh -o 3 -p 1
+//               mpirun -np 4 ex21p -m ../data/inline-quad.mesh -o 3 -p 2
+//               mpirun -np 4 ex21p -m ../data/inline-tet.mesh -o 2
+//               mpirun -np 4 ex21p -m ../data/inline-hex.mesh -o 2
+//               mpirun -np 4 ex21p -m ../data/inline-hex.mesh -o 2 -p 1
+//               mpirun -np 4 ex21p -m ../data/inline-hex.mesh -o 2 -p 2
+//               mpirun -np 4 ex21p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Description:  This is a version of Example 2p with a simple adaptive mesh
-//               refinement loop. The problem being solved is again the linear
-//               elasticity describing a multi-material cantilever beam.
-//               The problem is solved on a sequence of meshes which
-//               are locally refined in a conforming (triangles, tetrahedrons)
-//               or non-conforming (quadrilaterals, hexahedra) manner according
-//               to a simple ZZ error estimator.
+// Description:  This example code demonstrates the use of MFEM to define and
+//               solve simple complex-valued linear systems.  We implement three
+//               variants of a damped harmonic oscillator:
 //
-//               The example demonstrates MFEM's capability to work with both
-//               conforming and nonconforming refinements, in 2D and 3D, on
-//               linear and curved meshes. Interpolation of functions from
-//               coarse to fine meshes, as well as persistent GLVis
-//               visualization are also illustrated.
+//               1) A scalar H1 field
+//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
 //
-//               We recommend viewing Examples 2p and 6p before viewing this
-//               example.
+//               2) A vector H(Curl) field
+//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
+//
+//               3) A vector H(Div) field
+//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
+//
+//               In each case the field is driven by a forced oscillation, with
+//               angular frequency omega, imposed at the boundary or a portion
+//               of the boundary.
+//
+//               In electromagnetics the coefficients are typically named the
+//               permeability, mu = 1/a, permittivity, epsilon = b, and
+//               conductivity, sigma = c.  The user can specify these constants
+//               using either set of names.
+//
+//#define MFEM_STRUMPACK_SRC
 
-#include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include "mfem.hpp"
 
 using namespace std;
 using namespace mfem;
 
+static double mu_ = 1.0;
+static double epsilon_ = 1.0;
+static double sigma_ = 20.0;
+static double omega_ = 10.0;
+
+double u0_real_exact(const Vector &);
+double u0_imag_exact(const Vector &);
+
+void u1_real_exact(const Vector &, Vector &);
+void u1_imag_exact(const Vector &, Vector &);
+
+void u2_real_exact(const Vector &, Vector &);
+void u2_imag_exact(const Vector &, Vector &);
+
+bool check_for_inline_mesh(const char * mesh_file);
+
 int main(int argc, char *argv[])
 {
-   // 0. Initialize MPI.
+   // 1. Initialize MPI.
    int num_procs, myid;
    MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   MPI_Comm comm = MPI_COMM_WORLD;
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
 
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/beam-tri.mesh";
-   int serial_ref_levels = 0;
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/inline-quad.mesh";
+   int ser_ref_levels = 1;
+   int par_ref_levels = 1;
    int order = 1;
-   bool static_cond = false;
+   int prob = 0;
+   double freq = -1.0;
+   double a_coef = 0.0;
    bool visualization = 1;
+   bool herm_conv = true;
+   bool exact_sol = true;
+#ifdef MFEM_USE_STRUMPACK
+   bool strumpack = false;
+#endif
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
-   args.AddOption(&serial_ref_levels, "-rs", "--refine-serial",
-                  "Number of uniform serial refinements (before parallel"
-                  " partitioning)");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&prob, "-p", "--problem-type",
+                  "Choose from 0: H_1, 1: H(Curl), or 2: H(Div) "
+                  "damped harmonic oscillator.");
+   args.AddOption(&a_coef, "-a", "--stiffness-coef",
+                  "Stiffness coefficient (spring constant or 1/mu).");
+   args.AddOption(&epsilon_, "-b", "--mass-coef",
+                  "Mass coefficient (or epsilon).");
+   args.AddOption(&sigma_, "-c", "--damping-coef",
+                  "Damping coefficient (or sigma).");
+   args.AddOption(&mu_, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon_, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");
+   args.AddOption(&sigma_, "-sigma", "--conductivity",
+                  "Conductivity (or damping constant).");
+   args.AddOption(&freq, "-f", "--frequency",
+                  "Frequency (in Hz).");
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&strumpack, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use STRUMPACK's double complex linear solver.");
+#endif
+   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
+                  "--no-hermitian", "Use convention for Hermitian operators.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -78,289 +135,524 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");
-
-   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
+   if ( a_coef != 0.0 )
    {
-      cerr << "\nInput mesh should have at least two materials and "
-           << "two boundary attributes! (See schematic in ex2.cpp)\n"
-           << endl;
-      MPI_Finalize();
-      return 3;
+      mu_ = 1.0 / a_coef;
+   }
+   if ( freq > 0.0 )
+   {
+      omega_ = 2.0 * M_PI * freq;
    }
 
-   // 3. Refine the mesh before parallel partitioning. Since a NURBS mesh can
-   //    currently only be refined uniformly, we need to convert it to a
-   //    piecewise-polynomial curved mesh. First we refine the NURBS mesh a bit
-   //    more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext && serial_ref_levels == 0)
+   exact_sol = check_for_inline_mesh(mesh_file);
+   if (myid == 0 && exact_sol)
    {
-      serial_ref_levels = 2;
-   }
-   for (int i = 0; i < serial_ref_levels; i++)
-   {
-      mesh.UniformRefinement();
-   }
-   if (mesh.NURBSext)
-   {
-      mesh.SetCurvature(2);
-   }
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // 4. Define a finite element space on the mesh. The polynomial order is
-   //    one (linear) by default, but this can be changed on the command line.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec, dim);
-
-   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
-   //    the right-hand side of the FEM linear system. In this case, b_i equals
-   //    the boundary integral of f*phi_i where f represents a "pull down"
-   //    force on the Neumann part of the boundary and phi_i are the basis
-   //    functions in the finite element fespace. The force is defined by the
-   //    VectorArrayCoefficient object f, which is a vector of Coefficient
-   //    objects. The fact that f is non-zero on boundary attribute 2 is
-   //    indicated by the use of piece-wise constants coefficient for its last
-   //    component. We don't assemble the discrete problem yet, this will be
-   //    done in the main loop.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-   {
-      Vector pull_force(pmesh.bdr_attributes.Max());
-      pull_force = 0.0;
-      pull_force(1) = -1.0e-2;
-      f.Set(dim-1, new PWConstCoefficient(pull_force));
+      cout << "Identified an 'inline' mesh" << endl;
    }
 
-   ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+   ComplexOperator::Convention conv =
+      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 6. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the linear elasticity integrator with piece-wise
-   //    constants coefficient lambda and mu.
-   Vector lambda(pmesh.attributes.Max());
-   lambda = 1.0;
-   lambda(0) = lambda(1)*50;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(pmesh.attributes.Max());
-   mu = 1.0;
-   mu(0) = mu(1)*50;
-   PWConstCoefficient mu_func(mu);
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
 
-   ParBilinearForm a(&fespace);
-   BilinearFormIntegrator *integ =
-      new ElasticityIntegrator(lambda_func,mu_func);
-   a.AddDomainIntegrator(integ);
-   if (static_cond) { a.EnableStaticCondensation(); }
-
-   // 7. The solution vector x and the associated finite element grid function
-   //    will be maintained over the AMR iterations. We initialize it to zero.
-   Vector zero_vec(dim);
-   zero_vec = 0.0;
-   VectorConstantCoefficient zero_vec_coeff(zero_vec);
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking only
-   //    boundary attribute 1 from the mesh as essential and converting it to a
-   //    list of true dofs.  The conversion to true dofs will be done in the
-   //    main loop.
-   Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-   ess_bdr = 0;
-   ess_bdr[0] = 1;
-
-   // 9. GLVis visualization.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
-   //     recover a smoothed flux (stress) that is subtracted from the element
-   //     flux to get an error indicator. We need to supply the space for the
-   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
-   //     Here, tdim represents the number of components for a symmetric (dim x
-   //     dim) tensor.
-   const int tdim = dim*(dim+1)/2;
-   L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fespace(&pmesh, &flux_fec, tdim);
-   ParFiniteElementSpace smooth_flux_fespace(&pmesh, &fec, tdim);
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace,
-                                       smooth_flux_fespace);
-
-   // 11. A refiner selects and refines elements based on a refinement strategy.
-   //     The strategy here is to refine elements with errors larger than a
-   //     fraction of the maximum element error. Other strategies are possible.
-   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
-   refiner.SetTotalErrorFraction(0.7);
-
-   // 12. The main AMR loop. In each iteration we solve the problem on the
-   //     current mesh, visualize the solution, and refine the mesh.
-   const int max_dofs = 50000;
-   const int max_amr_itr = 20;
-   for (int it = 0; it <= max_amr_itr; it++)
+   // 4. Refine the serial mesh on all processors to increase the resolution.
+   for (int l = 0; l < ser_ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int l = 0; l < par_ref_levels; l++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define a parallel finite element space on the parallel
+   //    mesh. Here we use continuous Lagrange, Nedelec, or
+   //    Raviart-Thomas finite elements of the specified order.
+   if (dim == 1 && prob != 0 )
    {
-      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
       if (myid == 0)
       {
-         cout << "\nAMR iteration " << it << endl;
-         cout << "Number of unknowns: " << global_dofs << endl;
+         cout << "Switching to problem type 0, H1 basis functions, "
+              << "for 1 dimensional mesh." << endl;
       }
-
-      // 13. Assemble the stiffness matrix and the right-hand side.
-      a.Assemble();
-      b.Assemble();
-
-      // 14. Set Dirichlet boundary values in the GridFunction x.
-      //     Determine the list of Dirichlet true DOFs in the linear system.
-      Array<int> ess_tdof_list;
-      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-      // 15. Create the linear system: eliminate boundary conditions, constrain
-      //     hanging nodes and possibly apply other transformations. The system
-      //     will be solved for true (unconstrained) DOFs only.
-
-      HypreParMatrix A;
-      Vector B, X;
-      const int copy_interior = 1;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-
-      // 16. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-      //     preconditioner from hypre.
-      HypreBoomerAMG amg;
-      amg.SetPrintLevel(0);
-      // amg.SetSystemsOptions(dim); // optional
-      CGSolver pcg(A.GetComm());
-      pcg.SetPreconditioner(amg);
-      pcg.SetOperator(A);
-      pcg.SetRelTol(1e-6);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(3); // print the first and the last iterations only
-      pcg.Mult(B, X);
-
-      // 17. After solving the linear system, reconstruct the solution as a
-      //     finite element GridFunction. Constrained nodes are interpolated
-      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
-      a.RecoverFEMSolution(X, b, x);
-
-      // 18. Send solution by socket to the GLVis server.
-      if (visualization && it == 0)
-      {
-         sol_sock.open(vishost, visport);
-         sol_sock.precision(8);
-      }
-      if (visualization && sol_sock.good())
-      {
-         GridFunction nodes(&fespace), *nodes_p = &nodes;
-         pmesh.GetNodes(nodes);
-         nodes += x;
-         int own_nodes = 0;
-         pmesh.SwapNodes(nodes_p, own_nodes);
-         x.Neg(); // visualize the backward displacement
-         sol_sock << "parallel " << num_procs << ' ' << myid << '\n';
-         sol_sock << "solution\n" << pmesh << x << flush;
-         x.Neg();
-         pmesh.SwapNodes(nodes_p, own_nodes);
-         if (it == 0)
-         {
-            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
-         }
-         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
-                  << "pause" << endl;
-         if (myid == 0)
-         {
-            cout << "Visualization paused. "
-                 "Press <space> in the GLVis window to continue." << endl;
-         }
-      }
-
-      if (global_dofs > max_dofs)
-      {
-         if (myid == 0)
-         {
-            cout << "Reached the maximum number of dofs. Stop." << endl;
-         }
-         break;
-      }
-
-      // 19. Call the refiner to modify the mesh. The refiner calls the error
-      //     estimator to obtain element errors, then it selects elements to be
-      //     refined and finally it modifies the mesh. The Stop() method can be
-      //     used to determine if a stopping criterion was met.
-      refiner.Apply(pmesh);
-      if (refiner.Stop())
-      {
-         if (myid == 0)
-         {
-            cout << "Stopping criterion satisfied. Stop." << endl;
-         }
-         break;
-      }
-
-      // 20. Update the space to reflect the new state of the mesh. Also,
-      //     interpolate the solution x so that it lies in the new space but
-      //     represents the same function. This saves solver iterations later
-      //     since we'll have a good initial guess of x in the next step.
-      //     Internally, FiniteElementSpace::Update() calculates an
-      //     interpolation matrix which is then used by GridFunction::Update().
-      fespace.Update();
-      x.Update();
-
-      // 21. Load balance the mesh, and update the space and solution. Currently
-      //     available only for nonconforming meshes.
-      if (pmesh.Nonconforming())
-      {
-         pmesh.Rebalance();
-
-         // Update the space and the GridFunction. This time the update matrix
-         // redistributes the GridFunction among the processors.
-         fespace.Update();
-         x.Update();
-      }
-
-      // 21. Inform also the bilinear and linear forms that the space has
-      //     changed.
-      a.Update();
-      b.Update();
+      prob = 0;
    }
 
+   FiniteElementCollection *fec;
+   switch (prob)
    {
-      ostringstream mref_name, mesh_name, sol_name;
-      mref_name << "ex21p_reference_mesh." << setfill('0') << setw(6) << myid;
-      mesh_name << "ex21p_deformed_mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "ex21p_displacement." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ref_out(mref_name.str().c_str());
-      mesh_ref_out.precision(16);
-      pmesh.Print(mesh_ref_out);
-
-      ofstream mesh_out(mesh_name.str().c_str());
-      mesh_out.precision(16);
-      GridFunction nodes(&fespace), *nodes_p = &nodes;
-      pmesh.GetNodes(nodes);
-      nodes += x;
-      int own_nodes = 0;
-      pmesh.SwapNodes(nodes_p, own_nodes);
-      pmesh.Print(mesh_out);
-      pmesh.SwapNodes(nodes_p, own_nodes);
-
-      ofstream x_out(sol_name.str().c_str());
-      x_out.precision(16);
-      x.Save(x_out);
+      case 0:  fec = new H1_FECollection(order, dim);      break;
+      case 1:  fec = new ND_FECollection(order, dim);      break;
+      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
    }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    based on the type of mesh and the problem type.
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      if (exact_sol)
+      {
+         switch (prob)
+         {
+            case 0:   ess_bdr = 0; ess_bdr[0] = 1;  break;
+            default:  ess_bdr = 1; ess_bdr[2] = 0;  break;
+         }
+      }
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system.
+   ParComplexLinearForm b(fespace, conv);
+   b.Vector::operator=(0.0);
+
+   // 9. Define the solution vector u as a parallel finite element
+   //    grid function corresponding to fespace. Initialize u with
+   //    initial guess of 1+0i or the exact solution if it is known.
+   ParComplexGridFunction u(fespace);
+   ParComplexGridFunction * u_exact = NULL;
+   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
+
+   FunctionCoefficient u0_r(u0_real_exact);
+   FunctionCoefficient u0_i(u0_imag_exact);
+   VectorFunctionCoefficient u1_r(dim, u1_real_exact);
+   VectorFunctionCoefficient u1_i(dim, u1_imag_exact);
+   VectorFunctionCoefficient u2_r(dim, u2_real_exact);
+   VectorFunctionCoefficient u2_i(dim, u2_imag_exact);
+
+   ConstantCoefficient zeroCoef(0.0);
+   ConstantCoefficient oneCoef(1.0);
+
+   Vector zeroVec(dim); zeroVec = 0.0;
+   Vector  oneVec(dim);  oneVec = 0.0; oneVec[(prob==2)?(dim-1):0] = 1.0;
+   VectorConstantCoefficient zeroVecCoef(zeroVec);
+   VectorConstantCoefficient oneVecCoef(oneVec);
+
+   switch (prob)
+   {
+      case 0:
+         u.ProjectBdrCoefficient(oneCoef, zeroCoef, ess_bdr);
+         if (exact_sol) { u_exact->ProjectCoefficient(u0_r, u0_i); }
+         break;
+      case 1:
+         u.ProjectBdrCoefficientTangent(oneVecCoef, zeroVecCoef, ess_bdr);
+         if (exact_sol) { u_exact->ProjectCoefficient(u1_r, u1_i); }
+         break;
+      case 2:
+         u.ProjectBdrCoefficientNormal(oneVecCoef, zeroVecCoef, ess_bdr);
+         if (exact_sol) { u_exact->ProjectCoefficient(u2_r, u2_i); }
+         break;
+   }
+
+   if (visualization && exact_sol)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
+      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
+                 << "window_title 'Exact Real Part'" << flush;
+      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
+                 << "window_title 'Exact Imaginary Part'" << flush;
+   }
+
+   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
+   //     space corresponding to the damped harmonic oscillator operator
+   //     of the appropriate type:
+   //
+   //     0) A scalar H1 field
+   //        -Div(a Grad) - omega^2 b + i omega c
+   //
+   //     1) A vector H(Curl) field
+   //        Curl(a Curl) - omega^2 b + i omega c
+   //
+   //     2) A vector H(Div) field
+   //        -Grad(a Div) - omega^2 b + i omega c
+   //
+   ConstantCoefficient stiffnessCoef(1.0/mu_);
+   ConstantCoefficient massCoef(-omega_ * omega_ * epsilon_);
+   ConstantCoefficient lossCoef(omega_ * sigma_);
+   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
+
+   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
+   switch (prob)
+   {
+      case 0:
+         a->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
+                                NULL);
+         a->AddDomainIntegrator(new MassIntegrator(massCoef),
+                                new MassIntegrator(lossCoef));
+         break;
+      case 1:
+         a->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
+                                NULL);
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                                new VectorFEMassIntegrator(lossCoef));
+         break;
+      case 2:
+         a->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
+                                NULL);
+         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
+                                new VectorFEMassIntegrator(lossCoef));
+         break;
+   }
+
+   // 10a. Set up the parallel bilinear form for the preconditioner
+   //      corresponding to the appropriate operator if the STRUMPACK solver
+   //      has not been selected.
+   //
+   //      0) A scalar H1 field
+   //         -Div(a Grad) - omega^2 b + omega c
+   //
+   //      1) A vector H(Curl) field
+   //         Curl(a Curl) + omega^2 b + omega c
+   //
+   //      2) A vector H(Div) field
+   //         -Grad(a Div) - omega^2 b + omega c
+   //
+   ParBilinearForm *pcOp = NULL;
+#ifdef MFEM_USE_STRUMPACK
+   if (!strumpack)
+#endif
+   {
+      pcOp = new ParBilinearForm(fespace);
+      switch (prob)
+      {
+         case 0:
+            pcOp->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef));
+            pcOp->AddDomainIntegrator(new MassIntegrator(massCoef));
+            pcOp->AddDomainIntegrator(new MassIntegrator(lossCoef));
+            break;
+         case 1:
+            pcOp->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef));
+            pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(negMassCoef));
+            pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
+            break;
+         case 2:
+            pcOp->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef));
+            pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef));
+            pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
+            break;
+      }
+   }
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, etc.
+   a->Assemble();
+   if (pcOp) { pcOp->Assemble(); }
+
+   OperatorHandle A;
+   Vector B, U;
+
+   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;
+
+   OperatorHandle PCOp;
+   if (pcOp) { pcOp->FormSystemMatrix(ess_tdof_list, PCOp); }
+
+   if (myid == 0)
+   {
+      ComplexHypreParMatrix * Ahyp =
+         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
+
+      cout << "Size of linear system: "
+           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
+   }
+
+   // 12. Define and apply a parallel FGMRES solver for AU=B with a
+   //     block diagonal preconditioner based on the appropriate multigrid
+   //     preconditioner from hypre or simply use STRUMPACK.
+#ifdef MFEM_USE_STRUMPACK
+   if (!strumpack)
+#endif
+   {
+      Array<HYPRE_Int> blockTrueOffsets;
+      blockTrueOffsets.SetSize(3);
+      blockTrueOffsets[0] = 0;
+      blockTrueOffsets[1] = PCOp.Ptr()->Height();
+      blockTrueOffsets[2] = PCOp.Ptr()->Height();
+      blockTrueOffsets.PartialSum();
+
+      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
+
+      Operator * pc_r = NULL;
+      Operator * pc_i = NULL;
+
+      switch (prob)
+      {
+         case 0:
+            pc_r =
+               new HypreBoomerAMG(dynamic_cast<HypreParMatrix&>(*PCOp.Ptr()));
+            pc_i = new ScaledOperator(pc_r,
+                                      (conv == ComplexOperator::HERMITIAN) ?
+                                      1.0:-1.0);
+            break;
+         case 1:
+            pc_r = new HypreAMS(dynamic_cast<HypreParMatrix&>(*PCOp.Ptr()),
+                                fespace);
+            pc_i = new ScaledOperator(pc_r,
+                                      (conv == ComplexOperator::HERMITIAN) ?
+                                      1.0:-1.0);
+            break;
+         case 2:
+            if (dim == 2 )
+            {
+               pc_r = new HypreAMS(dynamic_cast<HypreParMatrix&>(*PCOp.Ptr()),
+                                   fespace);
+            }
+            else
+            {
+               pc_r = new HypreADS(dynamic_cast<HypreParMatrix&>(*PCOp.Ptr()),
+                                   fespace);
+            }
+            pc_i = new ScaledOperator(pc_r,
+                                      (conv == ComplexOperator::HERMITIAN) ?
+                                      1.0:-1.0);
+            break;
+      }
+      BDP.SetDiagonalBlock(0, pc_r);
+      BDP.SetDiagonalBlock(1, pc_i);
+      BDP.owns_blocks = 0;
+
+      FGMRESSolver fgmres(MPI_COMM_WORLD);
+      fgmres.SetPreconditioner(BDP);
+      fgmres.SetOperator(*A.Ptr());
+      fgmres.SetRelTol(1e-12);
+      fgmres.SetMaxIter(1000);
+      fgmres.SetPrintLevel(1);
+      fgmres.Mult(B, U);
+   }
+#ifdef MFEM_USE_STRUMPACK
+   else
+   {
+      ComplexHypreParMatrix * Ahyp =
+         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
+
+      STRUMPACKRowLocCmplxMatrix A_strmp(Ahyp->real(), Ahyp->imag());
+
+      STRUMPACKCmplxSolver strmp(argc, argv, comm);
+
+      strmp.SetPrintFactorStatistics(true);
+      strmp.SetPrintSolveStatistics(true);
+      // strmp.SetKrylovSolver(strumpack::KrylovSolver::AUTO); // core dump
+      strmp.SetKrylovSolver(strumpack::KrylovSolver::DIRECT); // core dump
+      // strmp.SetKrylovSolver(strumpack::KrylovSolver::REFINE); // core dump
+      // strmp.SetKrylovSolver(strumpack::KrylovSolver::PREC_GMRES); // index out of range asserts from strumpack::DenseMatrix
+      // strmp.SetKrylovSolver(strumpack::KrylovSolver::GMRES); // WORKS
+      strmp.SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strmp.SetOperator(A_strmp);
+      strmp.SetFromCommandLine();
+      strmp.Mult(B, U);
+   }
+#endif
+
+   // 13. Recover the parallel grid function corresponding to U. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(U, b, u);
+
+   if (exact_sol)
+   {
+      double err_r = -1.0;
+      double err_i = -1.0;
+
+      switch (prob)
+      {
+         case 0:
+            err_r = u.real().ComputeL2Error(u0_r);
+            err_i = u.imag().ComputeL2Error(u0_i);
+            break;
+         case 1:
+            err_r = u.real().ComputeL2Error(u1_r);
+            err_i = u.imag().ComputeL2Error(u1_i);
+            break;
+         case 2:
+            err_r = u.real().ComputeL2Error(u2_r);
+            err_i = u.imag().ComputeL2Error(u2_i);
+            break;
+      }
+
+      if ( myid == 0 )
+      {
+         cout << endl;
+         cout << "|| Re (u_h - u) ||_{L^2} = " << err_r << endl;
+         cout << "|| Im (u_h - u) ||_{L^2} = " << err_i << endl;
+         cout << endl;
+      }
+   }
+
+   // 14. Save the refined mesh and the solution in parallel. This output can be
+   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_r_name, sol_i_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
+      sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_r_ofs(sol_r_name.str().c_str());
+      ofstream sol_i_ofs(sol_i_name.str().c_str());
+      sol_r_ofs.precision(8);
+      sol_i_ofs.precision(8);
+      u.real().Save(sol_r_ofs);
+      u.imag().Save(sol_i_ofs);
+   }
+
+   // 15. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
+      sol_sock_r << "solution\n" << *pmesh << u.real()
+                 << "window_title 'Comp Real Part'" << flush;
+      sol_sock_i << "solution\n" << *pmesh << u.imag()
+                 << "window_title 'Comp Imaginary Part'" << flush;
+   }
+   if (visualization && exact_sol)
+   {
+      *u_exact -= u;
+
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
+      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
+      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
+                 << "window_title 'Exact-Comp Real Part'" << flush;
+      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
+                 << "window_title 'Exact-Comp Imaginary Part'" << flush;
+   }
+   if (visualization)
+   {
+      ParGridFunction u_t(fespace);
+      u_t = u.real();
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << u_t
+               << "window_title 'Harmonic Solution (t = 0.0 T)'"
+               << "pause\n" << flush;
+      if (myid == 0)
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      int num_frames = 32;
+      int i = 0;
+      while (sol_sock)
+      {
+         double t = (double)(i % num_frames) / num_frames;
+         ostringstream oss;
+         oss << "Harmonic Solution (t = " << t << " T)";
+
+         add(cos( 2.0 * M_PI * t), u.real(),
+             sin(-2.0 * M_PI * t), u.imag(), u_t);
+         sol_sock << "parallel " << num_procs << " " << myid << "\n";
+         sol_sock << "solution\n" << *pmesh << u_t
+                  << "window_title '" << oss.str() << "'" << flush;
+         i++;
+      }
+   }
+
+   // 16. Free the used memory.
+   delete a;
+   delete u_exact;
+   delete pcOp;
+   delete fespace;
+   delete fec;
+   delete pmesh;
 
    MPI_Finalize();
+
    return 0;
 }
+
+bool check_for_inline_mesh(const char * mesh_file)
+{
+   string file(mesh_file);
+   size_t p0 = file.find_last_of("/");
+   string s0 = file.substr((p0==string::npos)?0:(p0+1),7);
+   return s0 == "inline-";
+}
+
+complex<double> u0_exact(const Vector &x)
+{
+   int dim = x.Size();
+   complex<double> i(0.0, 1.0);
+   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
+   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
+   return std::exp(-i * kappa * x[dim - 1]);
+}
+
+double u0_real_exact(const Vector &x)
+{
+   return u0_exact(x).real();
+}
+
+double u0_imag_exact(const Vector &x)
+{
+   return u0_exact(x).imag();
+}
+
+void u1_real_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[0] = u0_real_exact(x);
+}
+
+void u1_imag_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[0] = u0_imag_exact(x);
+}
+
+void u2_real_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[dim-1] = u0_real_exact(x);
+}
+
+void u2_imag_exact(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+   v.SetSize(dim); v = 0.0; v[dim-1] = u0_imag_exact(x);
+}

examples/ex22.cpp

@@ -0,0 +1,310 @@
+//                                MFEM Example 22
+//
+// Compile with: make ex22
+//
+// Sample runs:  ex22
+//               ex22 -o 3
+//               ex22 -m ../data/beam-quad.mesh
+//               ex22 -m ../data/beam-quad.mesh -o 3
+//               ex22 -m ../data/beam-quad.mesh -o 3 -f 1
+//               ex22 -m ../data/beam-tet.mesh
+//               ex22 -m ../data/beam-tet.mesh -o 2
+//               ex22 -m ../data/beam-hex.mesh
+//               ex22 -m ../data/beam-hex.mesh -o 2
+//
+// Description:  This is a version of Example 2 with a simple adaptive mesh
+//               refinement loop. The problem being solved is again the linear
+//               elasticity describing a multi-material cantilever beam.
+//               The problem is solved on a sequence of meshes which
+//               are locally refined in a conforming (triangles, tetrahedrons)
+//               or non-conforming (quadrilaterals, hexahedra) manner according
+//               to a simple ZZ error estimator.
+//
+//               The example demonstrates MFEM's capability to work with both
+//               conforming and nonconforming refinements, in 2D and 3D, on
+//               linear and curved meshes. Interpolation of functions from
+//               coarse to fine meshes, as well as persistent GLVis
+//               visualization are also illustrated.
+//
+//               We recommend viewing Examples 2 and 6 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/beam-tri.mesh";
+   int order = 1;
+   bool static_cond = false;
+   int flux_averaging = 0;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&flux_averaging, "-f", "--flux-averaging",
+                  "Flux averaging: 0 - global, 1 - by mesh attribute.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");
+
+   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
+   {
+      cerr << "\nInput mesh should have at least two materials and "
+           << "two boundary attributes! (See schematic in ex2.cpp)\n"
+           << endl;
+      return 3;
+   }
+
+   // 3. Since a NURBS mesh can currently only be refined uniformly, we need to
+   //    convert it to a piecewise-polynomial curved mesh. First we refine the
+   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
+   if (mesh.NURBSext)
+   {
+      for (int i = 0; i < 2; i++)
+      {
+         mesh.UniformRefinement();
+      }
+      mesh.SetCurvature(2);
+   }
+
+   // 4. Define a finite element space on the mesh. The polynomial order is
+   //    one (linear) by default, but this can be changed on the command line.
+   H1_FECollection fec(order, dim);
+   FiniteElementSpace fespace(&mesh, &fec, dim);
+
+   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
+   //    the right-hand side of the FEM linear system. In this case, b_i equals
+   //    the boundary integral of f*phi_i where f represents a "pull down"
+   //    force on the Neumann part of the boundary and phi_i are the basis
+   //    functions in the finite element fespace. The force is defined by the
+   //    VectorArrayCoefficient object f, which is a vector of Coefficient
+   //    objects. The fact that f is non-zero on boundary attribute 2 is
+   //    indicated by the use of piece-wise constants coefficient for its last
+   //    component. We don't assemble the discrete problem yet, this will be
+   //    done in the main loop.
+   VectorArrayCoefficient f(dim);
+   for (int i = 0; i < dim-1; i++)
+   {
+      f.Set(i, new ConstantCoefficient(0.0));
+   }
+   {
+      Vector pull_force(mesh.bdr_attributes.Max());
+      pull_force = 0.0;
+      pull_force(1) = -1.0e-2;
+      f.Set(dim-1, new PWConstCoefficient(pull_force));
+   }
+
+   LinearForm b(&fespace);
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+
+   // 6. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the linear elasticity integrator with piece-wise
+   //    constants coefficient lambda and mu.
+   Vector lambda(mesh.attributes.Max());
+   lambda = 1.0;
+   lambda(0) = lambda(1)*50;
+   PWConstCoefficient lambda_func(lambda);
+   Vector mu(mesh.attributes.Max());
+   mu = 1.0;
+   mu(0) = mu(1)*50;
+   PWConstCoefficient mu_func(mu);
+
+   BilinearForm a(&fespace);
+   BilinearFormIntegrator *integ =
+      new ElasticityIntegrator(lambda_func,mu_func);
+   a.AddDomainIntegrator(integ);
+   if (static_cond) { a.EnableStaticCondensation(); }
+
+   // 7. The solution vector x and the associated finite element grid function
+   //    will be maintained over the AMR iterations. We initialize it to zero.
+   Vector zero_vec(dim);
+   zero_vec = 0.0;
+   VectorConstantCoefficient zero_vec_coeff(zero_vec);
+   GridFunction x(&fespace);
+   x = 0.0;
+
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking only
+   //    boundary attribute 1 from the mesh as essential and converting it to a
+   //    list of true dofs.  The conversion to true dofs will be done in the
+   //    main loop.
+   Array<int> ess_bdr(mesh.bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+
+   // 9. Connect to GLVis.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+   if (visualization)
+   {
+      sol_sock.open(vishost, visport);
+      sol_sock.precision(8);
+   }
+
+   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
+   //     recover a smoothed flux (stress) that is subtracted from the element
+   //     flux to get an error indicator. We need to supply the space for the
+   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
+   //     Here, tdim represents the number of components for a symmetric (dim x
+   //     dim) tensor.
+   const int tdim = dim*(dim+1)/2;
+   FiniteElementSpace flux_fespace(&mesh, &fec, tdim);
+   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
+   estimator.SetFluxAveraging(flux_averaging);
+
+   // 11. A refiner selects and refines elements based on a refinement strategy.
+   //     The strategy here is to refine elements with errors larger than a
+   //     fraction of the maximum element error. Other strategies are possible.
+   //     The refiner will call the given error estimator.
+   ThresholdRefiner refiner(estimator);
+   refiner.SetTotalErrorFraction(0.7);
+
+   // 12. The main AMR loop. In each iteration we solve the problem on the
+   //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 50000;
+   const int max_amr_itr = 20;
+   for (int it = 0; it <= max_amr_itr; it++)
+   {
+      int cdofs = fespace.GetTrueVSize();
+      cout << "\nAMR iteration " << it << endl;
+      cout << "Number of unknowns: " << cdofs << endl;
+
+      // 13. Assemble the stiffness matrix and the right-hand side.
+      a.Assemble();
+      b.Assemble();
+
+      // 14. Set Dirichlet boundary values in the GridFunction x.
+      //     Determine the list of Dirichlet true DOFs in the linear system.
+      Array<int> ess_tdof_list;
+      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      // 15. Create the linear system: eliminate boundary conditions, constrain
+      //     hanging nodes and possibly apply other transformations. The system
+      //     will be solved for true (unconstrained) DOFs only.
+      SparseMatrix A;
+      Vector B, X;
+      const int copy_interior = 1;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+
+#ifndef MFEM_USE_SUITESPARSE
+      // 16. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+      //     solve the linear system with PCG.
+      GSSmoother M(A);
+      PCG(A, M, B, X, 3, 2000, 1e-12, 0.0);
+#else
+      // 16. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
+      //     the linear system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(B, X);
+#endif
+
+      // 17. After solving the linear system, reconstruct the solution as a
+      //     finite element GridFunction. Constrained nodes are interpolated
+      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
+      a.RecoverFEMSolution(X, b, x);
+
+      // 18. Send solution by socket to the GLVis server.
+      if (visualization && sol_sock.good())
+      {
+         GridFunction nodes(&fespace), *nodes_p = &nodes;
+         mesh.GetNodes(nodes);
+         nodes += x;
+         int own_nodes = 0;
+         mesh.SwapNodes(nodes_p, own_nodes);
+         x.Neg(); // visualize the backward displacement
+         sol_sock << "solution\n" << mesh << x << flush;
+         x.Neg();
+         mesh.SwapNodes(nodes_p, own_nodes);
+         if (it == 0)
+         {
+            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
+         }
+         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
+                  << "pause" << endl;
+         cout << "Visualization paused. "
+              "Press <space> in the GLVis window to continue." << endl;
+      }
+
+      if (cdofs > max_dofs)
+      {
+         cout << "Reached the maximum number of dofs. Stop." << endl;
+         break;
+      }
+
+      // 19. Call the refiner to modify the mesh. The refiner calls the error
+      //     estimator to obtain element errors, then it selects elements to be
+      //     refined and finally it modifies the mesh. The Stop() method can be
+      //     used to determine if a stopping criterion was met.
+      refiner.Apply(mesh);
+      if (refiner.Stop())
+      {
+         cout << "Stopping criterion satisfied. Stop." << endl;
+         break;
+      }
+
+      // 20. Update the space to reflect the new state of the mesh. Also,
+      //     interpolate the solution x so that it lies in the new space but
+      //     represents the same function. This saves solver iterations later
+      //     since we'll have a good initial guess of x in the next step.
+      //     Internally, FiniteElementSpace::Update() calculates an
+      //     interpolation matrix which is then used by GridFunction::Update().
+      fespace.Update();
+      x.Update();
+
+      // 21. Inform also the bilinear and linear forms that the space has
+      //     changed.
+      a.Update();
+      b.Update();
+   }
+
+   {
+      ofstream mesh_ref_out("ex22_reference.mesh");
+      mesh_ref_out.precision(16);
+      mesh.Print(mesh_ref_out);
+
+      ofstream mesh_out("ex22_deformed.mesh");
+      mesh_out.precision(16);
+      GridFunction nodes(&fespace), *nodes_p = &nodes;
+      mesh.GetNodes(nodes);
+      nodes += x;
+      int own_nodes = 0;
+      mesh.SwapNodes(nodes_p, own_nodes);
+      mesh.Print(mesh_out);
+      mesh.SwapNodes(nodes_p, own_nodes);
+
+      ofstream x_out("ex22_displacement.sol");
+      x_out.precision(16);
+      x.Save(x_out);
+   }
+
+   return 0;
+}

examples/ex22p.cpp

@@ -0,0 +1,366 @@
+//                                MFEM Example 22
+//
+// Compile with: make ex22p
+//
+// Sample runs:  mpirun -np 4 ex22p
+//               mpirun -np 4 ex22p -o 3
+//               mpirun -np 4 ex22p -m ../data/beam-quad.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-quad.mesh -o 3
+//               mpirun -np 4 ex22p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-tet.mesh -o 2
+//               mpirun -np 4 ex22p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-hex.mesh -o 2
+//
+// Description:  This is a version of Example 2p with a simple adaptive mesh
+//               refinement loop. The problem being solved is again the linear
+//               elasticity describing a multi-material cantilever beam.
+//               The problem is solved on a sequence of meshes which
+//               are locally refined in a conforming (triangles, tetrahedrons)
+//               or non-conforming (quadrilaterals, hexahedra) manner according
+//               to a simple ZZ error estimator.
+//
+//               The example demonstrates MFEM's capability to work with both
+//               conforming and nonconforming refinements, in 2D and 3D, on
+//               linear and curved meshes. Interpolation of functions from
+//               coarse to fine meshes, as well as persistent GLVis
+//               visualization are also illustrated.
+//
+//               We recommend viewing Examples 2p and 6p before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/beam-tri.mesh";
+   int serial_ref_levels = 0;
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&serial_ref_levels, "-rs", "--refine-serial",
+                  "Number of uniform serial refinements (before parallel"
+                  " partitioning)");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");
+
+   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
+   {
+      cerr << "\nInput mesh should have at least two materials and "
+           << "two boundary attributes! (See schematic in ex2.cpp)\n"
+           << endl;
+      MPI_Finalize();
+      return 3;
+   }
+
+   // 3. Refine the mesh before parallel partitioning. Since a NURBS mesh can
+   //    currently only be refined uniformly, we need to convert it to a
+   //    piecewise-polynomial curved mesh. First we refine the NURBS mesh a bit
+   //    more and then project the curvature to quadratic Nodes.
+   if (mesh.NURBSext && serial_ref_levels == 0)
+   {
+      serial_ref_levels = 2;
+   }
+   for (int i = 0; i < serial_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   if (mesh.NURBSext)
+   {
+      mesh.SetCurvature(2);
+   }
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define a finite element space on the mesh. The polynomial order is
+   //    one (linear) by default, but this can be changed on the command line.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fec, dim);
+
+   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
+   //    the right-hand side of the FEM linear system. In this case, b_i equals
+   //    the boundary integral of f*phi_i where f represents a "pull down"
+   //    force on the Neumann part of the boundary and phi_i are the basis
+   //    functions in the finite element fespace. The force is defined by the
+   //    VectorArrayCoefficient object f, which is a vector of Coefficient
+   //    objects. The fact that f is non-zero on boundary attribute 2 is
+   //    indicated by the use of piece-wise constants coefficient for its last
+   //    component. We don't assemble the discrete problem yet, this will be
+   //    done in the main loop.
+   VectorArrayCoefficient f(dim);
+   for (int i = 0; i < dim-1; i++)
+   {
+      f.Set(i, new ConstantCoefficient(0.0));
+   }
+   {
+      Vector pull_force(pmesh.bdr_attributes.Max());
+      pull_force = 0.0;
+      pull_force(1) = -1.0e-2;
+      f.Set(dim-1, new PWConstCoefficient(pull_force));
+   }
+
+   ParLinearForm b(&fespace);
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
+
+   // 6. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the linear elasticity integrator with piece-wise
+   //    constants coefficient lambda and mu.
+   Vector lambda(pmesh.attributes.Max());
+   lambda = 1.0;
+   lambda(0) = lambda(1)*50;
+   PWConstCoefficient lambda_func(lambda);
+   Vector mu(pmesh.attributes.Max());
+   mu = 1.0;
+   mu(0) = mu(1)*50;
+   PWConstCoefficient mu_func(mu);
+
+   ParBilinearForm a(&fespace);
+   BilinearFormIntegrator *integ =
+      new ElasticityIntegrator(lambda_func,mu_func);
+   a.AddDomainIntegrator(integ);
+   if (static_cond) { a.EnableStaticCondensation(); }
+
+   // 7. The solution vector x and the associated finite element grid function
+   //    will be maintained over the AMR iterations. We initialize it to zero.
+   Vector zero_vec(dim);
+   zero_vec = 0.0;
+   VectorConstantCoefficient zero_vec_coeff(zero_vec);
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking only
+   //    boundary attribute 1 from the mesh as essential and converting it to a
+   //    list of true dofs.  The conversion to true dofs will be done in the
+   //    main loop.
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+
+   // 9. GLVis visualization.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+
+   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
+   //     recover a smoothed flux (stress) that is subtracted from the element
+   //     flux to get an error indicator. We need to supply the space for the
+   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
+   //     Here, tdim represents the number of components for a symmetric (dim x
+   //     dim) tensor.
+   const int tdim = dim*(dim+1)/2;
+   L2_FECollection flux_fec(order, dim);
+   ParFiniteElementSpace flux_fespace(&pmesh, &flux_fec, tdim);
+   ParFiniteElementSpace smooth_flux_fespace(&pmesh, &fec, tdim);
+   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace,
+                                       smooth_flux_fespace);
+
+   // 11. A refiner selects and refines elements based on a refinement strategy.
+   //     The strategy here is to refine elements with errors larger than a
+   //     fraction of the maximum element error. Other strategies are possible.
+   //     The refiner will call the given error estimator.
+   ThresholdRefiner refiner(estimator);
+   refiner.SetTotalErrorFraction(0.7);
+
+   // 12. The main AMR loop. In each iteration we solve the problem on the
+   //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 50000;
+   const int max_amr_itr = 20;
+   for (int it = 0; it <= max_amr_itr; it++)
+   {
+      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
+      if (myid == 0)
+      {
+         cout << "\nAMR iteration " << it << endl;
+         cout << "Number of unknowns: " << global_dofs << endl;
+      }
+
+      // 13. Assemble the stiffness matrix and the right-hand side.
+      a.Assemble();
+      b.Assemble();
+
+      // 14. Set Dirichlet boundary values in the GridFunction x.
+      //     Determine the list of Dirichlet true DOFs in the linear system.
+      Array<int> ess_tdof_list;
+      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      // 15. Create the linear system: eliminate boundary conditions, constrain
+      //     hanging nodes and possibly apply other transformations. The system
+      //     will be solved for true (unconstrained) DOFs only.
+
+      HypreParMatrix A;
+      Vector B, X;
+      const int copy_interior = 1;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+
+      // 16. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreBoomerAMG amg;
+      amg.SetPrintLevel(0);
+      // amg.SetSystemsOptions(dim); // optional
+      CGSolver pcg(A.GetComm());
+      pcg.SetPreconditioner(amg);
+      pcg.SetOperator(A);
+      pcg.SetRelTol(1e-6);
+      pcg.SetMaxIter(500);
+      pcg.SetPrintLevel(3); // print the first and the last iterations only
+      pcg.Mult(B, X);
+
+      // 17. After solving the linear system, reconstruct the solution as a
+      //     finite element GridFunction. Constrained nodes are interpolated
+      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
+      a.RecoverFEMSolution(X, b, x);
+
+      // 18. Send solution by socket to the GLVis server.
+      if (visualization && it == 0)
+      {
+         sol_sock.open(vishost, visport);
+         sol_sock.precision(8);
+      }
+      if (visualization && sol_sock.good())
+      {
+         GridFunction nodes(&fespace), *nodes_p = &nodes;
+         pmesh.GetNodes(nodes);
+         nodes += x;
+         int own_nodes = 0;
+         pmesh.SwapNodes(nodes_p, own_nodes);
+         x.Neg(); // visualize the backward displacement
+         sol_sock << "parallel " << num_procs << ' ' << myid << '\n';
+         sol_sock << "solution\n" << pmesh << x << flush;
+         x.Neg();
+         pmesh.SwapNodes(nodes_p, own_nodes);
+         if (it == 0)
+         {
+            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
+         }
+         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
+                  << "pause" << endl;
+         if (myid == 0)
+         {
+            cout << "Visualization paused. "
+                 "Press <space> in the GLVis window to continue." << endl;
+         }
+      }
+
+      if (global_dofs > max_dofs)
+      {
+         if (myid == 0)
+         {
+            cout << "Reached the maximum number of dofs. Stop." << endl;
+         }
+         break;
+      }
+
+      // 19. Call the refiner to modify the mesh. The refiner calls the error
+      //     estimator to obtain element errors, then it selects elements to be
+      //     refined and finally it modifies the mesh. The Stop() method can be
+      //     used to determine if a stopping criterion was met.
+      refiner.Apply(pmesh);
+      if (refiner.Stop())
+      {
+         if (myid == 0)
+         {
+            cout << "Stopping criterion satisfied. Stop." << endl;
+         }
+         break;
+      }
+
+      // 20. Update the space to reflect the new state of the mesh. Also,
+      //     interpolate the solution x so that it lies in the new space but
+      //     represents the same function. This saves solver iterations later
+      //     since we'll have a good initial guess of x in the next step.
+      //     Internally, FiniteElementSpace::Update() calculates an
+      //     interpolation matrix which is then used by GridFunction::Update().
+      fespace.Update();
+      x.Update();
+
+      // 21. Load balance the mesh, and update the space and solution. Currently
+      //     available only for nonconforming meshes.
+      if (pmesh.Nonconforming())
+      {
+         pmesh.Rebalance();
+
+         // Update the space and the GridFunction. This time the update matrix
+         // redistributes the GridFunction among the processors.
+         fespace.Update();
+         x.Update();
+      }
+
+      // 22. Inform also the bilinear and linear forms that the space has
+      //     changed.
+      a.Update();
+      b.Update();
+   }
+
+   {
+      ostringstream mref_name, mesh_name, sol_name;
+      mref_name << "ex22p_reference_mesh." << setfill('0') << setw(6) << myid;
+      mesh_name << "ex22p_deformed_mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "ex22p_displacement." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ref_out(mref_name.str().c_str());
+      mesh_ref_out.precision(16);
+      pmesh.Print(mesh_ref_out);
+
+      ofstream mesh_out(mesh_name.str().c_str());
+      mesh_out.precision(16);
+      GridFunction nodes(&fespace), *nodes_p = &nodes;
+      pmesh.GetNodes(nodes);
+      nodes += x;
+      int own_nodes = 0;
+      pmesh.SwapNodes(nodes_p, own_nodes);
+      pmesh.Print(mesh_out);
+      pmesh.SwapNodes(nodes_p, own_nodes);
+
+      ofstream x_out(sol_name.str().c_str());
+      x_out.precision(16);
+      x.Save(x_out);
+   }
+
+   MPI_Finalize();
+   return 0;
+}

examples/ex23.cpp

@@ -1,553 +0,0 @@
-//                               MFEM Example 23
-//
-// Compile with: make ex23
-//
-// Sample runs:
-//    ex23 -m ../data/periodic-segment.mesh -p 0 -r 2 -dt 0.005
-//    ex23 -m ../data/periodic-square.mesh -p 0 -r 2 -dt 0.01
-//    ex23 -m ../data/periodic-hexagon.mesh -p 0 -r 2 -dt 0.01
-//    ex23 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//    ex23 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//    ex23 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//    ex23 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//    ex23 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//    ex23 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
-//    ex23 -m ../data/disc-nurbs.mesh -p 2 -r 3 -dt 0.005 -tf 9
-//    ex23 -m ../data/disc-nurbs.mesh -p 2 -r 3 -dt 0.005 -tf 9 -d 0.05
-//    ex23 -m ../data/periodic-square.mesh -p 3 -r 4 -dt 0.0025 -tf 9 -vs 20
-//    ex23 -m ../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8
-//
-// Description:  This example code solves the time-dependent advection-diffusion
-//               equation
-//               du/dt - div(D grad(u)) + v.grad(u) = 0, where
-//               D is a diffusion coefficient,
-//               v is a given fluid velocity, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
-//
-//               This example is a merger of examples 9 and 14.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Choice for the problem setup. The fluid velocity, initial condition and
-// inflow boundary condition are chosen based on this parameter.
-int problem;
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-
-/** A time-dependent operator for the right-hand side of the ODE. The DG weak
-    form of du/dt = div(D grad(u))-v.grad(u) is
-    [M + dt (S - K)] du/dt = - S u + K u + b, where M, S, and K are the mass,
-    stiffness, and advection matrices, and b describes sources and the flow on
-    the boundary.
-    This can be written as a general ODE,
-    du/dt = A^{-1} (-S u + K u + b) with A = [M + dt (S - K)], and this class is
-    used to perform the implicit or explicit solve for du/dt. */
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   SparseMatrix &M, &S, &K;
-   SparseMatrix *A;
-   const Vector &b;
-
-   DSmoother M_prec;
-   CGSolver M_solver;
-
-   DSmoother *A_prec;
-   GMRESSolver *A_solver;
-   double dt;
-
-   mutable Vector z;
-
-   void initA(double dt);
-
-public:
-   FE_Evolution(SparseMatrix &_M, SparseMatrix &_S, SparseMatrix &_K,
-                const Vector &_b);
-
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &y);
-
-   virtual ~FE_Evolution() { delete A_solver; delete A_prec; delete A; }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   problem = 0;
-   const char *mesh_file = "../data/periodic-hexagon.mesh";
-   int ref_levels = 2;
-   int order = 3;
-   int ode_solver_type = 3;
-   double t_final = 10.0;
-   double d_coef = 0.01;
-   double dt = 0.01;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&d_coef, "-d", "--diff-coef",
-                  "Diffusion coefficient.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the serial mesh from the given mesh file on all processors. We can
-   //    handle geometrically periodic meshes in this code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 3. Define the ODE solver used for time integration. Several explicit
-   //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         return 3;
-   }
-
-   // 4. Refine the mesh in serial to increase the resolution. In this example
-   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-   //    a command-line parameter. If the mesh is of NURBS type, we convert it
-   //    to a (piecewise-polynomial) high-order mesh.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh.UniformRefinement();
-   }
-   if (mesh.NURBSext)
-   {
-      mesh.SetCurvature(max(order, 1));
-   }
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 5. Define the parallel discontinuous DG finite element space on the
-   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim);
-   FiniteElementSpace fes(&mesh, &fec);
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   // 6. Set up and assemble the parallel bilinear and linear forms (and the
-   //    parallel hypre matrices) corresponding to the DG discretization. The
-   //    DGTraceIntegrator involves integrals over mesh interior faces.
-   ConstantCoefficient diff_coef(d_coef);
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient u0(u0_function);
-
-   BilinearForm m(&fes);
-   m.AddDomainIntegrator(new MassIntegrator);
-
-   BilinearForm s(&fes);
-   s.AddDomainIntegrator(new DiffusionIntegrator(diff_coef));
-   s.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(diff_coef, sigma,
-                                                         kappa));
-   s.AddBdrFaceIntegrator(new DGDiffusionIntegrator(diff_coef, sigma, kappa));
-
-   BilinearForm k(&fes);
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   k.AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-   k.AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-
-   LinearForm b(&fes);
-   b.AddBdrFaceIntegrator(
-      new DGDirichletLFIntegrator(u0, diff_coef, sigma, kappa));
-
-   int skip_zeros = 0;
-   m.Assemble(skip_zeros);
-   m.Finalize(skip_zeros);
-   s.Assemble(skip_zeros);
-   s.Finalize(skip_zeros);
-   k.Assemble(skip_zeros);
-   k.Finalize(skip_zeros);
-   b.Assemble();
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u(&fes);
-   u.ProjectCoefficient(u0);
-
-   {
-      ofstream omesh("ex23.mesh");
-      omesh.precision(precision);
-      mesh.Print(omesh);
-      ofstream osol("ex23-init.gf");
-      osol.precision(precision);
-      u.Save(osol);
-   }
-
-   // Create data collection for solution output: either VisItDataCollection for
-   // ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example23", &mesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example23", &mesh);
-         dc->SetPrecision(precision);
-      }
-      dc->RegisterField("solution", &u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << mesh << u;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 8. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m.SpMat(), s.SpMat(), k.SpMat(), b);
-
-   double t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-      ode_solver->Step(u, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-
-         if (visualization)
-         {
-            sout << "solution\n" << mesh << u << flush;
-         }
-
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-      }
-   }
-
-   // 9. Save the final solution in parallel. This output can be viewed later
-   //    using GLVis: "glvis -np <np> -m ex23-mesh -g ex23-final".
-   {
-      ofstream osol("ex23-final.gf");
-      osol.precision(precision);
-      u.Save(osol);
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete dc;
-
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_S,
-                           SparseMatrix &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Height()),
-     M(_M), S(_S), K(_K), A(NULL), b(_b),
-     M_prec(M),
-     A_prec(NULL), A_solver(NULL), dt(-1.0), z(M.Height())
-{
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::initA(double _dt)
-{
-   if (fabs(dt - _dt) > 1e-4 * _dt)
-   {
-      delete A_solver;
-      delete A_prec;
-      delete A;
-
-      SparseMatrix * SK = Add(1.0, S, -1.0, K);
-      A = Add(1.0, M, _dt, *SK);
-      delete SK;
-      dt = _dt;
-
-      A_prec = new DSmoother(*A);
-      A_solver = new GMRESSolver;
-      A_solver->SetOperator(*A);
-      A_solver->SetPreconditioner(*A_prec);
-
-      A_solver->iterative_mode = false;
-      A_solver->SetRelTol(1e-9);
-      A_solver->SetAbsTol(0.0);
-      A_solver->SetMaxIter(100);
-      A_solver->SetPrintLevel(0);
-   }
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // y = M^{-1} (-S x + K x + b)
-   K.Mult(x, z);
-   S.AddMult(x, z, -1.0);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-void FE_Evolution::ImplicitSolve(const double _dt, const Vector &x, Vector &y)
-{
-   this->initA(_dt);
-
-   // y = (M + dt S - dt K)^{-1} (-S x + K x + b)
-   K.Mult(x, z);
-   S.AddMult(x, z, -1.0);
-   z += b;
-   A_solver->Mult(z, y);
-}
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-double u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return exp(-40.*pow(X(0)-0.5,2));
-            case 2:
-            case 3:
-            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const double f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      case 2:
-      case 3: return 0.0;
-   }
-   return 0.0;
-}

examples/ex23p.cpp

@@ -1,673 +0,0 @@
-//                       MFEM Example 23 - Parallel Version
-//
-// Compile with: make ex23p
-//
-// Sample runs:
-//    mpirun -np 4 ex23p -m ../data/periodic-segment.mesh -p 0 -dt 0.005
-//    mpirun -np 4 ex23p -m ../data/periodic-square.mesh -p 0 -dt 0.01
-//    mpirun -np 4 ex23p -m ../data/periodic-hexagon.mesh -p 0 -dt 0.01
-//    mpirun -np 4 ex23p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
-//    mpirun -np 4 ex23p -m ../data/periodic-hexagon.mesh -p 1 -dt 0.005 -tf 9
-//    mpirun -np 4 ex23p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9
-//    mpirun -np 4 ex23p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.004 -tf 9
-//    mpirun -np 4 ex23p -m ../data/star-mixed.mesh -p 1 -rp 1 -dt 0.004 -tf 9
-//    mpirun -np 4 ex23p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
-//    mpirun -np 4 ex23p -m ../data/disc-nurbs.mesh -p 2 -rp 1 -dt 0.005 -tf 9
-//    mpirun -np 4 ex23p -m ../data/disc-nurbs.mesh -p 3 -rp 1 -dt 0.005 -tf 9 -d 0.05
-//    mpirun -np 4 ex23p -m ../data/periodic-square.mesh -p 3 -rp 2 -dt 0.0025 -tf 9 -vs 20
-//    mpirun -np 4 ex23p -m ../data/periodic-cube.mesh -p 0 -o 2 -rp 1 -dt 0.01 -tf 8
-//
-// Description:  This example code solves the time-dependent advection-diffusion
-//               equation
-//               du/dt - div(D grad(u)) + v.grad(u) = 0, where
-//               D is a diffusion coefficient,
-//               v is a given fluid velocity, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
-//
-//               This example is a merger of examples 9 and 14.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Choice for the problem setup. The fluid velocity, initial condition and
-// inflow boundary condition are chosen based on this parameter.
-int problem, use_gmres;
-bool use_AIR;
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-struct AIR_parameters {
-   double distanceR;
-   std::string prerelax;
-   std::string postrelax;
-   int interp_type;
-   int relax_type;
-   int coarsen_type;
-   double strength_tolC;
-   double strength_tolR;
-   double filter_tolR;
-   double filterA_tol;
-};
-
-/** A time-dependent operator for the right-hand side of the ODE. The DG weak
-    form of du/dt = div(D grad(u))-v.grad(u) is
-    [M + dt (S - K)] du/dt = - S u + K u + b, where M, S, and K are the mass,
-    stiffness, and advection matrices, and b describes sources and the flow on
-    the boundary.
-    This can be written as a general ODE,
-    du/dt = A^{-1} (-S u + K u + b) with A = [M + dt (S - K)], and this class is
-    used to perform the implicit or explicit solve for du/dt. */
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   HypreParMatrix &M, &S, &K;
-   HypreParMatrix *A;
-   HypreParMatrix A_s;
-   const Vector &b;
-
-   HypreSmoother M_prec;
-   CGSolver M_solver;
-
-   HypreBoomerAMG *AMG_solver;
-   HypreGMRES *GMRES_solver;
-   double dt;
-   int blocksize;
-
-   mutable Vector z;
-
-public:
-   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_S, HypreParMatrix &_K,
-                const Vector &_b, int order);
-
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &y);
-
-   virtual ~FE_Evolution() { delete GMRES_solver; delete AMG_solver; delete A; }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   problem = 0;
-   use_gmres = true;
-   const char *mesh_file = "../data/periodic-hexagon.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 0;
-   int order = 3;
-   int ode_solver_type = 3;
-   double t_final = 10.0;
-   double d_coef = 0.01;
-   double dt = 0.01;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
-                  "\t   11 - Forward Euler, 12 - RK2, 13 - RK3 SSP, 14 - RK4.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&d_coef, "-d", "--diff-coef",
-                  "Diffusion coefficient.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the serial mesh from the given mesh file on all processors. We can
-   //    handle geometrically periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Define the ODE solver used for time integration. Several explicit
-   //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
-
-   // 5. Refine the mesh in serial to increase the resolution. In this example
-   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-   //    a command-line parameter. If the mesh is of NURBS type, we convert it
-   //    to a (piecewise-polynomial) high-order mesh.
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-   if (mesh->NURBSext)
-   {
-      mesh->SetCurvature(max(order, 1));
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // Get mesh size, compare with time step and diffusion coefficient. Use
-   // classical AMG for diffusion-dominated problems, and nonsymmetric AMG
-   // based on approximate ideal restriction (AIR) for advection dominated.
-   double h_min, h_max, k_min, k_max;
-   pmesh->GetCharacteristics(h_min, h_max, k_min, k_max);
-   if (dt > d_coef*h_max) use_AIR = true;
-   else use_AIR = true;
-   cout << "ratio = " << d_coef*h_max / dt << "\n";
-
-   // 7. Define the parallel discontinuous DG finite element space on the
-   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim);
-   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
-
-   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << global_vSize << endl;
-   }
-
-   // 8. Set up and assemble the parallel bilinear and linear forms (and the
-   //    parallel hypre matrices) corresponding to the DG discretization. The
-   //    DGTraceIntegrator involves integrals over mesh interior faces.
-   ConstantCoefficient diff_coef(d_coef);
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient u0(u0_function);
-
-   ParBilinearForm *m = new ParBilinearForm(fes);
-   m->AddDomainIntegrator(new MassIntegrator);
-
-   ParBilinearForm *s = new ParBilinearForm(fes);
-   s->AddDomainIntegrator(new DiffusionIntegrator(diff_coef));
-   s->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(diff_coef, sigma,
-                                                          kappa));
-   s->AddBdrFaceIntegrator(new DGDiffusionIntegrator(diff_coef, sigma, kappa));
-
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   k->AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-   k->AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-
-   ParLinearForm *b = new ParLinearForm(fes);
-   b->AddBdrFaceIntegrator(
-      new DGDirichletLFIntegrator(u0, diff_coef, sigma, kappa));
-
-   int skip_zeros = 0;
-   m->Assemble(skip_zeros);
-   m->Finalize(skip_zeros);
-   s->Assemble(skip_zeros);
-   s->Finalize(skip_zeros);
-   k->Assemble(skip_zeros);
-   k->Finalize(skip_zeros);
-   b->Assemble();
-
-   HypreParMatrix *M = m->ParallelAssemble();
-   HypreParMatrix *S = s->ParallelAssemble();
-   HypreParMatrix *K = k->ParallelAssemble();
-   HypreParVector *B = b->ParallelAssemble();
-
-   // 9. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   ParGridFunction *u = new ParGridFunction(fes);
-   u->ProjectCoefficient(u0);
-   HypreParVector *U = u->GetTrueDofs();
-
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "ex23-mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "ex23-init." << setfill('0') << setw(6) << myid;
-      ofstream omesh(mesh_name.str().c_str());
-      omesh.precision(precision);
-      pmesh->Print(omesh);
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u->Save(osol);
-   }
-
-   // Create data collection for solution output: either VisItDataCollection for
-   // ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example23-Parallel", pmesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example23-Parallel", pmesh);
-         dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
-      }
-      dc->RegisterField("solution", u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (myid == 0)
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-         visualization = false;
-         if (myid == 0)
-         {
-            cout << "GLVis visualization disabled.\n";
-         }
-      }
-      else
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout.precision(precision);
-         sout << "solution\n" << *pmesh << *u;
-         sout << "pause\n";
-         sout << flush;
-         if (myid == 0)
-            cout << "GLVis visualization paused."
-                 << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 10. Define the time-dependent evolution operator describing the ODE
-   //     right-hand side, and perform time-integration (looping over the time
-   //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*M, *S, *K, *B, order);
-
-   double t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-      ode_solver->Step(*U, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         if (myid == 0)
-         {
-            cout << "time step: " << ti << ", time: " << t << endl;
-         }
-
-         // 11. Extract the parallel grid function corresponding to the finite
-         //     element approximation U (the local solution on each processor).
-         *u = *U;
-
-         if (visualization)
-         {
-            sout << "parallel " << num_procs << " " << myid << "\n";
-            sout << "solution\n" << *pmesh << *u << flush;
-         }
-
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-      }
-   }
-
-   // 12. Save the final solution in parallel. This output can be viewed later
-   //     using GLVis: "glvis -np <np> -m ex23-mesh -g ex23-final".
-   {
-      *u = *U;
-      ostringstream sol_name;
-      sol_name << "ex23-final." << setfill('0') << setw(6) << myid;
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u->Save(osol);
-   }
-
-   // 13. Free the used memory.
-   delete U;
-   delete u;
-   delete B;
-   delete b;
-   delete K;
-   delete k;
-   delete S;
-   delete s;
-   delete M;
-   delete m;
-   delete fes;
-   delete pmesh;
-   delete ode_solver;
-   delete dc;
-
-   MPI_Finalize();
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_S,
-                           HypreParMatrix &_K, const Vector &_b, int order)
-   : TimeDependentOperator(_M.Height()),
-     M(_M), S(_S), K(_K), b(_b), GMRES_solver(NULL), AMG_solver(NULL),
-     M_prec(M), M_solver(M.GetComm()), A(NULL),
-     dt(-1.0), z(M.Height())
-{
-   M_prec.SetType(HypreSmoother::Jacobi);
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-
-   // DG block size given by (FEorder+1)^2 on square meshes.
-   blocksize = (order+1)*(order+1);
-}
-
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // y = M^{-1} (-S x + K x + b)
-   S.Mult(-1.0, x, 0.0, z);
-   K.Mult(1.0, x, 1.0, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-
-void FE_Evolution::ImplicitSolve(const double _dt, const Vector &x, Vector &y)
-{
-   if ((fabs(dt - _dt) > 1e-4 * _dt) || !A)
-   {
-      delete GMRES_solver;
-      delete AMG_solver;
-      delete A;
-
-      dt = _dt;
-      HypreParMatrix *SK = Add(1.0, S, -1.0, K);
-      A = Add(1.0, M, dt, *SK);
-      delete SK;
-
-      BlockInvScal(A, &A_s, NULL, NULL, blocksize, 0);
-
-      int print_level = 1;
-      AMG_solver = new HypreBoomerAMG(A_s);
-      AMG_solver->SetMaxLevels(50);      
-      if (use_AIR) {
-         AMG_solver->SetLAIROptions(1.5, "", "FFC", 0.1, 0.01, 0.0,
-                                    100, 3, 0.0, 10, -1, 1);
-                                // 100, 3, 0.0, 6, -1, 1);
-      }
-      else {
-         AMG_solver->SetInterpolation(0);
-         AMG_solver->SetCoarsening(6);
-         AMG_solver->SetAggressiveCoarsening(1);
-      }
-
-      if (use_gmres) {
-         GMRES_solver = new HypreGMRES(A_s);
-         GMRES_solver->SetTol(1e-12);
-         GMRES_solver->SetMaxIter(100);
-         GMRES_solver->SetPrintLevel(print_level);
-         GMRES_solver->SetPreconditioner(*AMG_solver);
-         GMRES_solver->iterative_mode = false;
-      }
-      else {
-         AMG_solver->SetPrintLevel(print_level);
-         AMG_solver->SetTol(1e-12);
-         AMG_solver->SetMaxIter(100);
-      }
-   }
-
-   // y = (M + dt S - dt K)^{-1} (-S x + K x + b)
-   S.Mult(-1.0, x, 0.0, z);
-   K.Mult(1.0, x, 1.0, z);
-   z += b;
-
-   // Scale block inverse to right hand side
-   HypreParVector b_s;
-   BlockInvScal(A, NULL, &z, &b_s, blocksize, 2);
-   GMRES_solver->Mult(b_s, y);
-
-}
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-double u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return exp(-40.*pow(X(0)-0.5,2));
-            case 2:
-            case 3:
-            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const double f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      case 2:
-      case 3: return 0.0;
-   }
-   return 0.0;
-}

examples/ex3p_sp.cpp

@@ -0,0 +1,334 @@
+//                       MFEM Example 3 - Parallel Version
+//
+// Compile with: make ex3p
+//
+// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh
+//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/fichera.mesh
+//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
+//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
+//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
+//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
+//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
+//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
+//
+// Description:  This example code solves a simple electromagnetic diffusion
+//               problem corresponding to the second order definite Maxwell
+//               equation curl curl E + E = f with boundary condition
+//               E x n = <given tangential field>. Here, we use a given exact
+//               solution E and compute the corresponding r.h.s. f.
+//               We discretize with Nedelec finite elements in 2D or 3D.
+//
+//               The example demonstrates the use of H(curl) finite element
+//               spaces with the curl-curl and the (vector finite element) mass
+//               bilinear form, as well as the computation of discretization
+//               error when the exact solution is known. Static condensation is
+//               also illustrated.
+//
+//               We recommend viewing examples 1-2 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, E, and r.h.s., f. See below for implementation.
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+int dim;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/beam-tet.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+#ifdef MFEM_USE_STRUMPACK
+   bool use_strumpack = false;
+#endif
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use STRUMPACK's double complex linear solver.");
+#endif
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+   kappa = freq * M_PI;
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   pmesh->ReorientTetMesh();
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 7. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh->bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 8. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
+   //    basis functions in the finite element fespace.
+   VectorFunctionCoefficient f(sdim, f_exact);
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
+   b->Assemble();
+
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
+   VectorFunctionCoefficient E(sdim, E_exact);
+   x.ProjectCoefficient(E);
+
+   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
+   //     mass domain integrators.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(-1.0);
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+
+   // 11. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   HypreParMatrix A;
+   Vector B, X;
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+   }
+
+   StopWatch chrono;
+   chrono.Clear();
+   chrono.Start();
+
+#ifdef MFEM_USE_STRUMPACK
+   if (use_strumpack)
+   {
+      Operator * Arow = new STRUMPACKRowLocMatrix(A);
+
+      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+      strumpack->SetPrintFactorStatistics(true);
+      strumpack->SetPrintSolveStatistics(false);
+      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
+      // strumpack->SetSymmetricPattern(true);
+      strumpack->SetOperator(*Arow);
+      strumpack->SetFromCommandLine();
+      //Solver * precond = strumpack;
+
+      strumpack->Mult(B, X);
+
+      delete strumpack;
+      delete Arow;
+   }
+   else
+#endif
+   {
+      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+      //     preconditioner from hypre.
+      ParFiniteElementSpace *prec_fespace =
+         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+      HypreSolver *ams = new HypreAMS(A, prec_fespace);
+      HyprePCG *pcg = new HyprePCG(A);
+      pcg->SetTol(1e-12);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(2);
+      pcg->SetPreconditioner(*ams);
+      pcg->Mult(B, X);
+
+      delete pcg;
+      delete ams;
+   }
+
+   chrono.Stop();
+   cout << "Solver time " << chrono.RealTime() << endl;
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a->RecoverFEMSolution(X, *b, x);
+
+   // 14. Compute and print the L^2 norm of the error.
+   {
+      double err = x.ComputeL2Error(E);
+      if (myid == 0)
+      {
+         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+      }
+   }
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   delete a;
+   delete sigma;
+   delete muinv;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   MPI_Finalize();
+
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}

examples/ex6.cpp

@@ -16,9 +16,9 @@
 //               ex6 -m ../data/amr-quad.mesh
 //
 // Device sample runs:
-//               ex6 -pa -d cuda
-//               ex6 -pa -d occa-cuda
-//               ex6 -pa -d raja-omp
+//             > ex6 -pa -d cuda
+//             > ex6 -pa -d occa-cuda
+//             > ex6 -pa -d raja-omp
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
@@ -49,7 +49,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
    bool visualization = true;
 
    OptionsParser args(argc, argv);
@@ -59,7 +59,7 @@ int main(int argc, char *argv[])
                   "Finite element order (polynomial degree).");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
@@ -72,19 +72,14 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
    Mesh mesh(mesh_file, 1, 1);
    int dim = mesh.Dimension();
    int sdim = mesh.SpaceDimension();
 
-   // 4. Since a NURBS mesh can currently only be refined uniformly, we need to
+   // 3. Since a NURBS mesh can currently only be refined uniformly, we need to
    //    convert it to a piecewise-polynomial curved mesh. First we refine the
    //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
    if (mesh.NURBSext)
@@ -96,11 +91,15 @@ int main(int argc, char *argv[])
       mesh.SetCurvature(2);
    }
 
-   // 5. Define a finite element space on the mesh. The polynomial order is
+   // 4. Define a finite element space on the mesh. The polynomial order is
    //    one (linear) by default, but this can be changed on the command line.
    H1_FECollection fec(order, dim);
    FiniteElementSpace fespace(&mesh, &fec);
 
+   // 5. Set device config parameters from the command line options.
+   Device::Configure(device);
+   Device::Print();
+
    // 6. As in Example 1, we set up bilinear and linear forms corresponding to
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
@@ -169,7 +168,8 @@ int main(int argc, char *argv[])
       x.ProjectBdrCoefficient(zero, ess_bdr);
       fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-      // 15. Assemble the stiffness matrix.
+      // 15. Switch to the device and assemble the stiffness matrix.
+      Device::Enable();
       a.Assemble();
 
       // 16. Create the linear system: eliminate boundary conditions, constrain
@@ -204,6 +204,7 @@ int main(int argc, char *argv[])
       // 18. After solving the linear system, reconstruct the solution as a
       //     finite element GridFunction. Constrained nodes are interpolated
       //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
+      Device::Disable();
       a.RecoverFEMSolution(X, b, x);
 
       // 19. Send solution by socket to the GLVis server.

examples/ex6p.cpp

@@ -16,9 +16,9 @@
 //               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
 //
 // Device sample runs:
-//               mpirun -np 4 ex6p -pa -d cuda
-//               mpirun -np 4 ex6p -pa -d occa-cuda
-//               mpirun -np 4 ex6p -pa -d raja-omp
+//             > mpirun -np 4 ex6p -pa -d cuda
+//             > mpirun -np 4 ex6p -pa -d occa-cuda
+//             > mpirun -np 4 ex6p -pa -d raja-omp
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
@@ -55,7 +55,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
    bool visualization = true;
 
    OptionsParser args(argc, argv);
@@ -65,7 +65,7 @@ int main(int argc, char *argv[])
                   "Finite element order (polynomial degree).");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
@@ -85,19 +85,14 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
    int sdim = mesh->SpaceDimension();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
    //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
    //    sure that the mesh is non-conforming.
    if (mesh->NURBSext)
@@ -107,7 +102,7 @@ int main(int argc, char *argv[])
    }
    mesh->EnsureNCMesh();
 
-   // 6. Define a parallel mesh by partitioning the serial mesh.
+   // 5. Define a parallel mesh by partitioning the serial mesh.
    //    Once the parallel mesh is defined, the serial mesh can be deleted.
    ParMesh pmesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
@@ -117,11 +112,15 @@ int main(int argc, char *argv[])
    Array<int> ess_bdr(pmesh.bdr_attributes.Max());
    ess_bdr = 1;
 
-   // 7. Define a finite element space on the mesh. The polynomial order is
+   // 6. Define a finite element space on the mesh. The polynomial order is
    //    one (linear) by default, but this can be changed on the command line.
    H1_FECollection fec(order, dim);
    ParFiniteElementSpace fespace(&pmesh, &fec);
 
+   // 7. Set device config parameters from the command line options.
+   Device::Configure(device);
+   if (myid == 0) { Device::Print(); }
+
    // 8. As in Example 1p, we set up bilinear and linear forms corresponding to
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
@@ -201,10 +200,11 @@ int main(int argc, char *argv[])
       fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
       b.Assemble();
 
-      // 15. Assemble the stiffness matrix. Note that MFEM doesn't care at this
-      //     point that the mesh is nonconforming and parallel.  The FE space is
-      //     considered 'cut' along hanging edges/faces, and also across
-      //     processor boundaries.
+      // 15. Switch to the device and assemble the stiffness matrix. Note that
+      //     MFEM doesn't care at this point that the mesh is nonconforming and
+      //     parallel.  The FE space is considered 'cut' along hanging
+      //     edges/faces, and also across processor boundaries.
+      Device::Enable();
       a.Assemble();
 
       // 16. Create the parallel linear system: eliminate boundary conditions.
@@ -232,6 +232,7 @@ int main(int argc, char *argv[])
       // 18. Switch back to the host and extract the parallel grid function
       //     corresponding to the finite element approximation X. This is the
       //     local solution on each processor.
+      Device::Disable();
       a.RecoverFEMSolution(X, b, x);
 
       // 19. Send the solution by socket to a GLVis server.

examples/ex9p.cpp

@@ -37,7 +37,7 @@ using namespace mfem;
 
 // Choice for the problem setup. The fluid velocity, initial condition and
 // inflow boundary condition are chosen based on this parameter.
-int problem, trisolve, use_gmres;
+int problem;
 
 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v);
@@ -51,19 +51,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
-struct AIR_parameters {
-   double distanceR;
-   std::string prerelax;
-   std::string postrelax;
-   int interp_type;
-   int relax_type;
-   int coarsen_type;
-   double strength_tolC;
-   double strength_tolR;
-   double filter_tolR;
-   double filterA_tol;
-};
-
 
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -73,33 +60,19 @@ struct AIR_parameters {
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   HypreParMatrix &M, &K, *A, A_s;
+   HypreParMatrix &M, &K;
    const Vector &b;
    HypreSmoother M_prec;
    CGSolver M_solver;
 
-   // Preconditioner/solvers for A
-   HypreBoomerAMG *AMG_solver;
-   HypreGMRES     *GMRES_solver;
-   HypreTriSolve *preconditioner;
-   AIR_parameters &AIR;
-
-   double dt;
-   int blocksize;
-
    mutable Vector z;
 
 public:
-   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b,
-                int order, AIR_parameters &_AIR);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);
 
-   /** Solve the Backward-Euler equation: d = f(x + dt*d, t+dt), where u_t = f(x,t).
-       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
    virtual void Mult(const Vector &x, Vector &y) const;
 
-   virtual ~FE_Evolution();
-
+   virtual ~FE_Evolution() { }
 };
 
 
@@ -113,28 +86,21 @@ int main(int argc, char *argv[])
 
    // 2. Parse command-line options.
    problem = 0;
-   use_gmres = 0;
-   trisolve = 0;
    const char *mesh_file = "../data/periodic-hexagon.mesh";
    int ser_ref_levels = 2;
    int par_ref_levels = 0;
    int order = 3;
-   int ode_solver_type = 3;
+   int ode_solver_type = 4;
    double t_final = 10.0;
    double dt = 0.01;
    bool visualization = true;
    bool visit = false;
    bool binary = false;
    int vis_steps = 5;
-   int basis_type = 1;
 
    int precision = 8;
    cout.precision(precision);
 
-    AIR_parameters AIR = {1, "", "FA", 100, 10, 10, 0.1, 0.01, 0.0, 1e-4};
-    const char* temp_prerelax = "";
-    const char* temp_postrelax = "FA";
-
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
                   "Mesh file to use.");
@@ -153,32 +119,6 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
-   args.AddOption(&basis_type, "-b", "--basis-type",
-                  "DG finite element basis type. 0 for G-Leg, 1 for G-Lob.");
-   args.AddOption(&use_gmres, "-gmres", "--use-gmres",
-                  "Boolean to use GMRES as solver (default with AIR preconditioning).");
-   args.AddOption(&trisolve, "-trisolve", "--precond-trisolve",
-                  "Precondition GMRES with an on-processor triangular solve.");
-   args.AddOption(&(AIR.distanceR), "-Ad", "--AIR-distance",
-                  "Distance restriction neighborhood for AIR.");
-   args.AddOption(&(AIR.interp_type), "-Ai", "--AIR-interpolation",
-                  "Index for hypre interpolation routine.");
-   args.AddOption(&(AIR.coarsen_type), "-Ac", "--AIR-coarsen_type",
-                  "Index for hypre coarsening routine.");
-   args.AddOption(&(AIR.strength_tolC), "-AsC", "--AIR-strengthC",
-                  "Theta value determining strong connections for AIR (coarsen_type).");
-   args.AddOption(&(AIR.strength_tolR), "-AsR", "--AIR-strengthR",
-                  "Theta value determining strong connections for AIR (restriction).");
-   args.AddOption(&(AIR.filter_tolR), "-AfR", "--AIR-filterR",
-                  "Theta value eliminating small entries in restriction (after building).");
-   args.AddOption(&(AIR.filterA_tol), "-Af", "--AIR-filter",
-                  "Theta value to eliminate small connections in AIR hierarchy. Use -1 to specify O(h).");
-   args.AddOption(&(AIR.relax_type), "-Ar", "--AIR-relaxation",
-                  "Index for hypre relaxation routine.");
-   args.AddOption(&temp_prerelax, "-Ar1", "--AIR-prerelax",
-                  "String denoting prerelaxation scheme; e.g., FCC.");
-   args.AddOption(&temp_postrelax, "-Ar2", "--AIR-postrelax",
-                  "String denoting postrelaxation scheme; e.g., FFC.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -191,9 +131,6 @@ int main(int argc, char *argv[])
    args.AddOption(&vis_steps, "-vs", "--visualization-steps",
                   "Visualize every n-th timestep.");
    args.Parse();
-   AIR.prerelax = std::string(temp_prerelax);
-   AIR.postrelax = std::string(temp_postrelax);
-   if (trisolve) use_gmres = 1;
    if (!args.Good())
    {
       if (myid == 0)
@@ -218,21 +155,11 @@ int main(int argc, char *argv[])
    ODESolver *ode_solver = NULL;
    switch (ode_solver_type)
    {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Explicit methods
-      case 11: ode_solver = new ForwardEulerSolver; break;
-      case 12: ode_solver = new RK2Solver(1.0); break;
-      case 13: ode_solver = new RK3SSPSolver; break;
-      case 14: ode_solver = new RK4Solver; break;
-      case 15: ode_solver = new GeneralizedAlphaSolver(0.5); break;
-      case 16: ode_solver = new RK6Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
       default:
          if (myid == 0)
          {
@@ -268,10 +195,8 @@ int main(int argc, char *argv[])
    }
 
    // 7. Define the parallel discontinuous DG finite element space on the
-   //    parallel refined mesh of the given polynomial order. Basis_type=1
-   //    gives Gauss-Lobatto quadrature points, which are preferable for
-   //    nonsymmetric AMG implict solves. 
-   DG_FECollection fec(order, dim, basis_type);
+   //    parallel refined mesh of the given polynomial order.
+   DG_FECollection fec(order, dim);
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
    HYPRE_Int global_vSize = fes->GlobalTrueVSize();
@@ -389,7 +314,7 @@ int main(int argc, char *argv[])
    // 10. Define the time-dependent evolution operator describing the ODE
    //     right-hand side, and perform time-integration (looping over the time
    //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*M, *K, *B, order, AIR);
+   FE_Evolution adv(*M, *K, *B);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -462,11 +387,9 @@ int main(int argc, char *argv[])
 
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
-                           const Vector &_b, int order,
-                           AIR_parameters &_AIR)
-   : TimeDependentOperator(_M.Height()), A(NULL), AMG_solver(NULL),
-     GMRES_solver(NULL), preconditioner(NULL), M(_M), K(_K), b(_b),
-     M_solver(M.GetComm()), z(_M.Height()), AIR(_AIR)
+                           const Vector &_b)
+   : TimeDependentOperator(_M.Height()),
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
 {
    M_prec.SetType(HypreSmoother::Jacobi);
    M_solver.SetPreconditioner(M_prec);
@@ -477,23 +400,8 @@ FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
    M_solver.SetAbsTol(0.0);
    M_solver.SetMaxIter(100);
    M_solver.SetPrintLevel(0);
-
-   // DG block size given by (FEorder+1)^2 on square meshes.
-   blocksize = (order+1)*(order+1);
-   dt = -1;
 }
 
-
-FE_Evolution::~FE_Evolution()
-{
-   BlockInvScal(NULL, NULL, NULL, NULL, 0, -1);
-   if (A) delete A;
-   if (AMG_solver) delete AMG_solver;
-   if (GMRES_solver) delete GMRES_solver;
-   if (preconditioner) delete preconditioner;
-}
-
-
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
    // y = M^{-1} (K x + b)
@@ -503,74 +411,6 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
 }
 
 
-// Solve the equation:
-//    u_t = M^{-1}(Ku + b), 
-// by solving associated linear system
-//    (M - dt*K) d = K*u + b
-void FE_Evolution::ImplicitSolve(const double dt_, const Vector &u, Vector &du_dt)
-{
-   // if A is NULL or dt has changed since A was built, rebuild matrix and solver.
-   if ( (fabs(dt - dt_) > 1e-4 * dt) || !A ) {
-      delete GMRES_solver;
-      delete AMG_solver;
-      delete preconditioner;
-      delete A;
-
-      dt = dt_;
-      A = HypreParMatrixAdd(1.0, M, -1.0*dt, K);
-
-      // Scale A by block-diagonal inverse
-      BlockInvScal(A, &A_s, NULL, NULL, blocksize, 0);
-
-      int print_level = 1;
-      if (!trisolve) {
-         AMG_solver = new HypreBoomerAMG(A_s);
-         AMG_solver->SetLAIROptions(AIR.distanceR, AIR.prerelax, AIR.postrelax,
-                                    AIR.strength_tolC, AIR.strength_tolR, AIR.filter_tolR,
-                                    AIR.interp_type, AIR.relax_type, AIR.filterA_tol,
-                                    AIR.coarsen_type, -1, 1);  
-         AMG_solver->SetMaxLevels(50);
-         if (use_gmres) {
-            GMRES_solver = new HypreGMRES(A_s);
-            GMRES_solver->SetTol(1e-12);
-            GMRES_solver->SetMaxIter(100);
-            GMRES_solver->SetPrintLevel(print_level);
-            GMRES_solver->SetPreconditioner(*AMG_solver);
-            GMRES_solver->iterative_mode = false;
-         }
-         else {
-            AMG_solver->SetPrintLevel(print_level);
-            AMG_solver->SetTol(1e-12);
-            AMG_solver->SetMaxIter(100);
-         }
-      }
-      else {
-         preconditioner = new HypreTriSolve();
-         GMRES_solver = new HypreGMRES(A_s);
-         GMRES_solver->SetTol(1e-12);
-         GMRES_solver->SetMaxIter(100);
-         GMRES_solver->SetPrintLevel(print_level);
-         GMRES_solver->SetPreconditioner(*preconditioner);
-         GMRES_solver->SetZeroInintialIterate();
-         GMRES_solver->iterative_mode = false;
-      }
-   }
-
-   K.Mult(u, z);
-   z += b;
-
-   // scale the rhs and solve system 
-   HypreParVector z_s;
-   BlockInvScal(A, NULL, &z, &z_s, blocksize, 2);
-   if (use_gmres){
-      GMRES_solver->Mult(z_s, du_dt);
-   }
-   else {
-      AMG_solver->Mult(z_s, du_dt);
-   }
-}
-
-
 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
 {

examples/makefile

@@ -22,9 +22,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
 SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
-  ex18 ex19 ex20 ex21 ex23
+  ex18 ex19 ex20 ex21 ex22
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
- ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p MFEM_adv ex23TRp exETRp
+ ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -117,17 +117,12 @@ clean-build:
 
 clean-exec:
 	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
-	@rm -rf Example5* Example9* Example15* Example16* Example23*
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.*
+	@rm -rf Example5* Example9* Example15* Example16*
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
 	@rm -f deformed.* velocity.* elastic_energy.* mode_*
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*
 	@rm -f vortex-mesh.* vortex.mesh vortex-?-init.* vortex-?-final.*
 	@rm -f deformation.* pressure.*
 	@rm -f ex20.dat ex20p_?????.dat gnuplot_ex20.inp gnuplot_ex20p.inp
-<<<<<<< HEAD
 	@rm -f ex22*.mesh ex22*.sol ex22p_*.*
-	@rm -f ex23.mesh ex23-mesh.* ex23-init.* ex23-final.*
-=======
-	@rm -f ex21*.mesh ex21*.sol ex21p_*.*
->>>>>>> f8a3a379d13841c6e63b9d8fbc868aa325b8afc0

examples/petsc/ex10p.cpp

@@ -27,11 +27,8 @@
 //               method HyperelasticOperator::ImplicitSolve is the only
 //               requirement for high-order implicit (SDIRK) time integration.
 //               If using PETSc to solve the nonlinear problem, use the option
-//               files provided (see rc_ex10p, rc_ex10p_mf, rc_ex10p_mfop) that
-//               customize the Newton-Krylov method.
-//               When option --jfnk is used, PETSc will use a Jacobian-free
-//               Newton-Krylov method, using a user-defined preconditioner
-//               constructed with the PetscPreconditionerFactory class.
+//               file provided (rc_ex10p) that customizes the
+//               Newton-Krylov method.
 //
 //               We recommend viewing examples 2 and 9 before viewing this
 //               example.
@@ -89,15 +86,12 @@ protected:
    Solver *J_solver;
    /// Preconditioner for the Jacobian solve in the Newton method
    Solver *J_prec;
-   /// Preconditioner factory for JFNK
-   PetscPreconditionerFactory *J_factory;
 
    mutable Vector z; // auxiliary vector
 
 public:
    HyperelasticOperator(ParFiniteElementSpace &f, Array<int> &ess_bdr,
-                        double visc, double mu, double K,
-                        bool use_petsc, bool petsc_use_jfnk);
+                        double visc, double mu, double K, bool use_petsc);
 
    /// Compute the right-hand side of the ODE system.
    virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
@@ -142,21 +136,8 @@ public:
    virtual Operator &GetGradient(const Vector &k) const;
 
    virtual ~ReducedSystemOperator();
-
 };
 
-/** Auxiliary class to provide preconditioners for matrix-free methods */
-class PreconditionerFactory : public PetscPreconditionerFactory
-{
-private:
-   // const ReducedSystemOperator& op; // unused for now (generates warning)
-
-public:
-   PreconditionerFactory(const ReducedSystemOperator& op_, const string& name_)
-      : PetscPreconditionerFactory(name_) /* , op(op_) */ {}
-   virtual mfem::Solver* NewPreconditioner(const mfem::OperatorHandle&);
-   virtual ~PreconditionerFactory() {}
-};
 
 /** Function representing the elastic energy density for the given hyperelastic
     model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */
@@ -206,7 +187,6 @@ int main(int argc, char *argv[])
    int vis_steps = 1;
    bool use_petsc = true;
    const char *petscrc_file = "";
-   bool petsc_use_jfnk = false;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -241,9 +221,6 @@ int main(int argc, char *argv[])
                   "Use or not PETSc to solve the nonlinear system.");
    args.AddOption(&petscrc_file, "-petscopts", "--petscopts",
                   "PetscOptions file to use.");
-   args.AddOption(&petsc_use_jfnk, "-jfnk", "--jfnk", "-no-jfnk",
-                  "--no-jfnk",
-                  "Use JFNK with user-defined preconditioner factory.");
    args.Parse();
    if (!args.Good())
    {
@@ -367,8 +344,7 @@ int main(int argc, char *argv[])
    // 9. Initialize the hyperelastic operator, the GLVis visualization and print
    //    the initial energies.
    HyperelasticOperator *oper = new HyperelasticOperator(fespace, ess_bdr, visc,
-                                                         mu, K, use_petsc,
-                                                         petsc_use_jfnk);
+                                                         mu, K, use_petsc);
 
    socketstream vis_v, vis_w;
    if (visualization)
@@ -544,7 +520,7 @@ Operator &ReducedSystemOperator::GetGradient(const Vector &k) const
    add(*v, dt, k, w);
    add(*x, dt, w, z);
    localJ->Add(dt*dt, H->GetLocalGradient(z));
-   // if we are using PETSc, the HypreParCSR Jacobian will be converted to
+   // if we are using PETSc, the HypreParCSR jacobian will be converted to
    // PETSc's AIJ on the fly
    Jacobian = M->ParallelAssemble(localJ);
    delete localJ;
@@ -561,8 +537,7 @@ ReducedSystemOperator::~ReducedSystemOperator()
 
 HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
                                            Array<int> &ess_bdr, double visc,
-                                           double mu, double K, bool use_petsc,
-                                           bool use_petsc_factory)
+                                           double mu, double K, bool use_petsc)
    : TimeDependentOperator(2*f.TrueVSize(), 0.0), fespace(f),
      M(&fespace), S(&fespace), H(&fespace),
      viscosity(visc), M_solver(f.GetComm()),
@@ -615,8 +590,6 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
       J_minres->SetPreconditioner(*J_prec);
       J_solver = J_minres;
 
-      J_factory = NULL;
-
       newton_solver.iterative_mode = false;
       newton_solver.SetSolver(*J_solver);
       newton_solver.SetOperator(*reduced_oper);
@@ -627,20 +600,12 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
    }
    else
    {
-      // if using PETSc, we create the same solver (Newton + MINRES + Jacobi)
+      // if using PETSc, we create the same solver (NEWTON+MINRES+Jacobi)
       // by command line options (see rc_ex10p)
       J_solver = NULL;
       J_prec = NULL;
-      J_factory = NULL;
       pnewton_solver = new PetscNonlinearSolver(f.GetComm(),
                                                 *reduced_oper);
-
-      // we can setup a factory to construct a "physics-based" preconditioner
-      if (use_petsc_factory)
-      {
-         J_factory = new PreconditionerFactory(*reduced_oper, "JFNK preconditioner");
-         pnewton_solver->SetPreconditionerFactory(J_factory);
-      }
       pnewton_solver->SetPrintLevel(1); // print Newton iterations
       pnewton_solver->SetRelTol(rel_tol);
       pnewton_solver->SetAbsTol(0.0);
@@ -726,26 +691,12 @@ HyperelasticOperator::~HyperelasticOperator()
 {
    delete J_solver;
    delete J_prec;
-   delete J_factory;
    delete reduced_oper;
    delete model;
    delete Mmat;
    delete pnewton_solver;
 }
 
-// This method gets called every time we need a preconditioner "oh"
-// contains the PetscParMatrix that wraps the operator constructed in
-// the GetGradient() method (see also PetscSolver::SetJacobianType()).
-// In this example, we just return a customizable PetscPreconditioner
-// using that matrix. However, the OperatorHandle argument can be
-// ignored, and any "physics-based" solver can be constructed since we
-// have access to the HyperElasticOperator class.
-Solver* PreconditionerFactory::NewPreconditioner(const mfem::OperatorHandle& oh)
-{
-   PetscParMatrix *pP;
-   oh.Get(pP);
-   return new PetscPreconditioner(*pP,"jfnk_");
-}
 
 double ElasticEnergyCoefficient::Eval(ElementTransformation &T,
                                       const IntegrationPoint &ip)
@@ -759,8 +710,8 @@ double ElasticEnergyCoefficient::Eval(ElementTransformation &T,
 
 void InitialDeformation(const Vector &x, Vector &y)
 {
-   // set the initial configuration to be the same as the reference,
-   // stress free, configuration
+   // set the initial configuration to be the same as the reference, stress
+   // free, configuration
    y = x;
 }
 

examples/petsc/makefile

@@ -84,10 +84,6 @@ EX9_E_ARGS       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts r
 EX9_ES_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
 EX9_IS_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
 EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
-EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
-EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
-EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
-
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
@@ -111,9 +107,6 @@ ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_IS_ARGS))
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/rc_ex10p_jfnk

@@ -1,5 +0,0 @@
-# matrix-free Jacobian action, preconditioner constructed using PetscPreconditionerFactory
--snes_monitor
--snes_mf_operator
--ksp_type minres
--jfnk_pc_type jacobi

examples/petsc/rc_ex10p_mf

@@ -1,4 +0,0 @@
-# matrix free -> no preconditioner
--snes_monitor
--snes_mf
--ksp_type minres

examples/petsc/rc_ex10p_mfop

@@ -1,5 +0,0 @@
-# matrix-free Jacobian action, preconditioner constructed from the matrix obtained by the GetGradient() method
--snes_monitor
--snes_mf_operator
--ksp_type minres
--pc_type jacobi

fem/CMakeLists.txt

@@ -13,8 +13,7 @@ set(SRCS
   bilinearform.cpp
   bilinearform_ext.cpp
   bilininteg.cpp
-  bilininteg_diffusion.cpp
-  bilininteg_mass.cpp
+  bilininteg_ext.cpp
   coefficient.cpp
   datacollection.cpp
   eltrans.cpp
@@ -38,6 +37,7 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
+  bilininteg_ext.hpp
   coefficient.hpp
   datacollection.hpp
   eltrans.hpp

fem/bilinearform.cpp

@@ -55,7 +55,7 @@ void BilinearForm::AllocMat()
 
    int *I = dof_dof.GetI();
    int *J = dof_dof.GetJ();
-   double *data = new double[I[height]];
+   double *data = mfem::New<double>(I[height]);
 
    mat = new SparseMatrix(I, J, data, height, height, true, true, true);
    *mat = 0.0;
@@ -122,7 +122,11 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
    switch (assembly)
    {
       case AssemblyLevel::FULL:
-         // ext = new FABilinearFormExtension(this);
+         if (Device::IsEnabled())
+         {
+            mfem_error("Full assembly not supported yet in device mode!");
+            // ext = new FABilinearFormExtension(this);
+         }
          // Use the original BilinearForm implementation for now
          break;
       case AssemblyLevel::ELEMENT:
@@ -294,33 +298,6 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
    }
 }
 
-void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (bbfi.Size())
-   {
-      const FiniteElement &be = *fes->GetBE(i);
-      ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
-      {
-         bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      fes->GetBdrElementVDofs(i, vdofs);
-      elmat.SetSize(vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void BilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleElementMatrix(i, elmat, vdofs, skip_zeros);
-}
-
 void BilinearForm::AssembleElementMatrix(
    int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
 {
@@ -343,12 +320,6 @@ void BilinearForm::AssembleElementMatrix(
    }
 }
 
-void BilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleBdrElementMatrix(i, elmat, vdofs, skip_zeros);
-}
-
 void BilinearForm::AssembleBdrElementMatrix(
    int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
 {
@@ -373,6 +344,11 @@ void BilinearForm::AssembleBdrElementMatrix(
 
 void BilinearForm::Assemble(int skip_zeros)
 {
+   if (Device::IsEnabled() && (assembly != AssemblyLevel::PARTIAL))
+   {
+      mfem_error("Chosen assembly level not supported yet in device mode!");
+   }
+
    if (ext)
    {
       ext->Assemble();
@@ -612,14 +588,14 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
                                     Vector &b, OperatorHandle &A, Vector &X,
                                     Vector &B, int copy_interior)
 {
-   const SparseMatrix *P = fes->GetConformingProlongation();
-
    if (ext)
    {
       ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
       return;
    }
 
+   const SparseMatrix *P = fes->GetConformingProlongation();
+
    FormSystemMatrix(ess_tdof_list, A);
 
    // Transform the system and perform the elimination in B, based on the
@@ -645,8 +621,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       {
          // A, X and B point to the same data as mat, x and b
          EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.NewDataAndSize(x.GetData(), x.Size());
+         B.NewDataAndSize(b.GetData(), b.Size());
          if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
       }
    }
@@ -747,10 +723,6 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,
       else
       {
          // X and x point to the same data
-
-         // If the validity flags of X's Memory were changed (e.g. if it was
-         // moved to device memory) then we need to tell x about that.
-         x.SyncMemory(X);
       }
    }
    else // non-conforming space
@@ -1049,13 +1021,9 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    extern_bfs = 1;
 
    // Copy the pointers to the integrators
-   dbfi = mbf->dbfi;
-   bbfi = mbf->bbfi;
-   tfbfi = mbf->tfbfi;
-   btfbfi = mbf->btfbfi;
-
-   bbfi_marker = mbf->bbfi_marker;
-   btfbfi_marker = mbf->btfbfi_marker;
+   dom = mbf->dom;
+   bdr = mbf->bdr;
+   skt = mbf->skt;
 }
 
 double & MixedBilinearForm::Elem (int i, int j)
@@ -1109,42 +1077,22 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
 
 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   dbfi.Append (bfi);
+   dom.Append (bfi);
 }
 
 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                                               Array<int> &bdr_marker)
-{
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   bdr.Append (bfi);
 }
 
 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   tfbfi.Append (bfi);
-}
-
-void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi)
-{
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
-                                                  Array<int> &bdr_marker)
-{
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(&bdr_marker);
+   skt.Append (bfi);
 }
 
 void MixedBilinearForm::Assemble (int skip_zeros)
 {
+   int i, k;
    Array<int> tr_vdofs, te_vdofs;
    ElementTransformation *eltrans;
    DenseMatrix elemmat;
@@ -1156,75 +1104,48 @@ void MixedBilinearForm::Assemble (int skip_zeros)
       mat = new SparseMatrix(height, width);
    }
 
-   if (dbfi.Size())
+   if (dom.Size())
    {
-      for (int i = 0; i < test_fes -> GetNE(); i++)
+      for (i = 0; i < test_fes -> GetNE(); i++)
       {
          trial_fes -> GetElementVDofs (i, tr_vdofs);
          test_fes  -> GetElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetElementTransformation (i);
-         for (int k = 0; k < dbfi.Size(); k++)
+         for (k = 0; k < dom.Size(); k++)
          {
-            dbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                               *test_fes  -> GetFE(i),
-                                               *eltrans, elemmat);
+            dom[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                              *test_fes  -> GetFE(i),
+                                              *eltrans, elemmat);
             mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
 
-   if (bbfi.Size())
+   if (bdr.Size())
    {
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (i = 0; i < test_fes -> GetNBE(); i++)
       {
-         if (bbfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *bbfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < test_fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
          trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
          test_fes  -> GetBdrElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetBdrElementTransformation (i);
-         for (int k = 0; k < bbfi.Size(); k++)
+         for (k = 0; k < bdr.Size(); k++)
          {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-            bbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
-                                               *test_fes  -> GetBE(i),
-                                               *eltrans, elemmat);
+            bdr[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                              *test_fes  -> GetBE(i),
+                                              *eltrans, elemmat);
             mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
 
-   if (tfbfi.Size())
+   if (skt.Size())
    {
       FaceElementTransformations *ftr;
       Array<int> te_vdofs2;
       const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
 
       int nfaces = mesh->GetNumFaces();
-      for (int i = 0; i < nfaces; i++)
+      for (i = 0; i < nfaces; i++)
       {
          ftr = mesh->GetFaceElementTransformations(i);
          trial_fes->GetFaceVDofs(i, tr_vdofs);
@@ -1244,70 +1165,14 @@ void MixedBilinearForm::Assemble (int skip_zeros)
             // want to actually make a fake element.
             test_fe2 = test_fe1;
          }
-         for (int k = 0; k < tfbfi.Size(); k++)
+         for (int k = 0; k < skt.Size(); k++)
          {
-            tfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                         *ftr, elemmat);
+            skt[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
+                                       *ftr, elemmat);
             mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
-
-   if (btfbfi.Size())
-   {
-      FaceElementTransformations *ftr;
-      Array<int> te_vdofs2;
-      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
-
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < btfbfi.Size(); k++)
-      {
-         if (btfbfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *btfbfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary trace face integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < trial_fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         ftr = mesh->GetBdrFaceTransformations(i);
-         if (ftr)
-         {
-            trial_fes->GetFaceVDofs(i, tr_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
-            trial_face_fe = trial_fes->GetFaceElement(i);
-            test_fe1 = test_fes->GetFE(ftr->Elem1No);
-            // The test_fe2 object is really a dummy and not used on the
-            // boundaries, but we can't dereference a NULL pointer, and we don't
-            // want to actually make a fake element.
-            test_fe2 = test_fe1;
-            for (int k = 0; k < btfbfi.Size(); k++)
-            {
-               if (btfbfi_marker[k] &&
-                   (*btfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-               btfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                             *ftr, elemmat);
-               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
-            }
-         }
-      }
-   }
 }
 
 void MixedBilinearForm::ConformingAssemble()
@@ -1336,93 +1201,8 @@ void MixedBilinearForm::ConformingAssemble()
    width = mat->Width();
 }
 
-
-void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (dbfi.Size())
-   {
-      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
-      const FiniteElement &test_fe = *test_fes->GetFE(i);
-      ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
-      {
-         dbfi[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      trial_fes->GetElementVDofs(i, trial_vdofs);
-      test_fes->GetElementVDofs(i, test_vdofs);
-      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (bbfi.Size())
-   {
-      const FiniteElement &trial_be = *trial_fes->GetBE(i);
-      const FiniteElement &test_be = *test_fes->GetBE(i);
-      ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
-      {
-         bbfi[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-      test_fes->GetBdrElementVDofs(i, test_vdofs);
-      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleElementMatrix(i, elmat, trial_vdofs, test_vdofs, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
-{
-   trial_fes->GetElementVDofs(i, trial_vdofs);
-   test_fes->GetElementVDofs(i, test_vdofs);
-   if (mat == NULL)
-   {
-      mat = new SparseMatrix(height, width);
-   }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleBdrElementMatrix(i, elmat, trial_vdofs, test_vdofs, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
-{
-   trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-   test_fes->GetBdrElementVDofs(i, test_vdofs);
-   if (mat == NULL)
-   {
-      mat = new SparseMatrix(height, width);
-   }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
-}
-
 void MixedBilinearForm::EliminateTrialDofs (
-   const Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
+   Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
 {
    int i, j, k;
    Array<int> tr_vdofs, cols_marker (trial_fes -> GetVSize());
@@ -1445,12 +1225,12 @@ void MixedBilinearForm::EliminateTrialDofs (
 }
 
 void MixedBilinearForm::EliminateEssentialBCFromTrialDofs (
-   const Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
+   Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
 {
    mat -> EliminateCols (marked_vdofs, &sol, &rhs);
 }
 
-void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
+void MixedBilinearForm::EliminateTestDofs (Array<int> &bdr_attr_is_ess)
 {
    int i, j, k;
    Array<int> te_vdofs;
@@ -1484,10 +1264,9 @@ MixedBilinearForm::~MixedBilinearForm()
    if (!extern_bfs)
    {
       int i;
-      for (i = 0; i < dbfi.Size(); i++) { delete dbfi[i]; }
-      for (i = 0; i < bbfi.Size(); i++) { delete bbfi[i]; }
-      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
-      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
+      for (i = 0; i < dom.Size(); i++) { delete dom[i]; }
+      for (i = 0; i < bdr.Size(); i++) { delete bdr[i]; }
+      for (i = 0; i < skt.Size(); i++) { delete skt[i]; }
    }
 }
 
@@ -1504,7 +1283,7 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
       mat = new SparseMatrix(height, width);
    }
 
-   if (dbfi.Size() > 0)
+   if (dom.Size() > 0)
    {
       for (int i = 0; i < test_fes->GetNE(); i++)
       {
@@ -1514,17 +1293,17 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
          dom_fe = trial_fes->GetFE(i);
          ran_fe = test_fes->GetFE(i);
 
-         dbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < dbfi.Size(); j++)
+         dom[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < dom.Size(); j++)
          {
-            dbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            dom[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
             totelmat += elmat;
          }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
       }
    }
 
-   if (tfbfi.Size())
+   if (skt.Size())
    {
       const int nfaces = test_fes->GetMesh()->GetNumFaces();
       for (int i = 0; i < nfaces; i++)
@@ -1535,10 +1314,10 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
          dom_fe = trial_fes->GetFaceElement(i);
          ran_fe = test_fes->GetFaceElement(i);
 
-         tfbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < tfbfi.Size(); j++)
+         skt[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < skt.Size(); j++)
          {
-            tfbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            skt[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
             totelmat += elmat;
          }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);

fem/bilinearform.hpp

@@ -413,49 +413,9 @@ public:
    void FreeElementMatrices()
    { delete element_matrices; element_matrices = NULL; }
 
-   /// Compute the element matrix of the given element
-   /** The element matrix is computed by calling the domain integrators
-       or the one stored internally by a prior call of ComputeElementMatrices()
-       is returned when available.
-   */
    void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Compute the boundary element matrix of the given boundary element
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The flag @a skip_zeros skips the zero
-       elements of the matrix, unless they are breaking the symmetry of
-       the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              int skip_zeros = 1);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The vdofs of the element are returned
-       in @a vdofs. The flag @a skip_zeros skips the zero elements of the
-       matrix, unless they are breaking the symmetry of the system matrix.
-   */
    void AssembleElementMatrix(int i, const DenseMatrix &elmat,
                               Array<int> &vdofs, int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The vdofs of the element
-       are returned in @a vdofs. The flag @a skip_zeros skips the zero elements
-       of the matrix, unless they are breaking the symmetry of the system matrix.
-   */
    void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
                                  Array<int> &vdofs, int skip_zeros = 1);
 
@@ -553,26 +513,16 @@ protected:
    FiniteElementSpace *trial_fes, ///< Not owned
                       *test_fes;  ///< Not owned
 
-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #dom, #bdr, and
+       #skt are owned by another MixedBilinearForm. */
    int extern_bfs;
 
    /// Domain integrators.
-   Array<BilinearFormIntegrator*> dbfi;
-
+   Array<BilinearFormIntegrator*> dom;
    /// Boundary integrators.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker;///< Entries are not owned.
-
+   Array<BilinearFormIntegrator*> bdr;
    /// Trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> tfbfi;
-
-   /// Boundary trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> btfbfi;
-   Array<Array<int>*>             btfbfi_marker;///< Entries are not owned.
-
-   DenseMatrix elemmat;
-   Array<int>  trial_vdofs, test_vdofs;
+   Array<BilinearFormIntegrator*> skt;
 
 private:
    /// Copy construction is not supported; body is undefined.
@@ -636,10 +586,6 @@ public:
    /// Adds a boundary integrator. Assumes ownership of @a bfi.
    void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);
 
-   /// Adds a boundary integrator. Assumes ownership of @a bfi.
-   void AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                               Array<int> &bdr_marker);
-
    /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
 
        This type of integrator assembles terms over all faces of the mesh using
@@ -647,32 +593,14 @@ public:
        test space. */
    void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi);
 
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi,
-                                   Array<int> &bdr_marker);
-
    /// Access all integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &dom; }
 
    /// Access all integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
-   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &bdr; }
 
    /// Access all integrators added with AddTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetTFBFI() { return &tfbfi; }
-
-   /// Access all integrators added with AddBdrTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBTFBFI() { return &btfbfi; }
-   /** @brief Access all boundary markers added with AddBdrTraceFaceIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }
+   Array<BilinearFormIntegrator*> *GetTFBFI() { return &skt; }
 
    void operator=(const double a) { *mat = a; }
 
@@ -685,59 +613,13 @@ public:
        MixedBilinearForm becomes an operator on the conforming FE spaces. */
    void ConformingAssemble();
 
-   /// Compute the element matrix of the given element
-   void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Compute the boundary element matrix of the given boundary element
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The flag @a skip_zeros skips the zero
-       elements of the matrix, unless they are breaking the symmetry of
-       the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              int skip_zeros = 1);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The vdofs of the element are returned
-       in @a trial_vdofs and @a test_vdofs. The flag @a skip_zeros skips
-       the zero elements of the matrix, unless they are breaking the symmetry
-       of the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              Array<int> &trial_vdofs, Array<int> &test_vdofs,
-                              int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The vdofs of the element
-       are returned in @a trial_vdofs and @a test_vdofs. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 Array<int> &trial_vdofs, Array<int> &test_vdofs,
-                                 int skip_zeros = 1);
-
-   void EliminateTrialDofs(const Array<int> &bdr_attr_is_ess,
+   void EliminateTrialDofs(Array<int> &bdr_attr_is_ess,
                            const Vector &sol, Vector &rhs);
 
-   void EliminateEssentialBCFromTrialDofs(const Array<int> &marked_vdofs,
+   void EliminateEssentialBCFromTrialDofs(Array<int> &marked_vdofs,
                                           const Vector &sol, Vector &rhs);
 
-   virtual void EliminateTestDofs(const Array<int> &bdr_attr_is_ess);
+   virtual void EliminateTestDofs(Array<int> &bdr_attr_is_ess);
 
    void Update();
 
@@ -802,7 +684,7 @@ public:
    { AddTraceFaceIntegrator(di); }
 
    /// Access all interpolators added with AddDomainInterpolator().
-   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDI() { return &dom; }
 
    /** @brief Construct the internal matrix representation of the discrete
        linear operator. */

fem/bilinearform_ext.cpp

@@ -36,18 +36,16 @@ const Operator *BilinearFormExtension::GetRestriction() const
 
 
 // Data and methods for partially-assembled bilinear forms
-PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
-   : BilinearFormExtension(form),
-     trialFes(a->FESpace()), testFes(a->FESpace())
+PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form) :
+   BilinearFormExtension(form),
+   trialFes(a->FESpace()), testFes(a->FESpace()),
+   localX(trialFes->GetNE() * trialFes->GetFE(0)->GetDof() * trialFes->GetVDim()),
+   localY( testFes->GetNE() * testFes->GetFE(0)->GetDof() * testFes->GetVDim()),
+   elem_restrict(new ElemRestriction(*a->FESpace())) { }
+
+PABilinearFormExtension::~PABilinearFormExtension()
 {
-   elem_restrict_lex = trialFes->GetElementRestriction(
-                          ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_lex)
-   {
-      localX.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
-   }
+   delete elem_restrict;
 }
 
 void PABilinearFormExtension::Assemble()
@@ -56,7 +54,7 @@ void PABilinearFormExtension::Assemble()
    const int integratorCount = integrators.Size();
    for (int i = 0; i < integratorCount; ++i)
    {
-      integrators[i]->AssemblePA(*a->FESpace());
+      integrators[i]->Assemble(*a->FESpace());
    }
 }
 
@@ -66,13 +64,12 @@ void PABilinearFormExtension::Update()
    height = width = fes->GetVSize();
    trialFes = fes;
    testFes = fes;
-   elem_restrict_lex = trialFes->GetElementRestriction(
-                          ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_lex)
-   {
-      localX.SetSize(elem_restrict_lex->Height());
-      localY.SetSize(elem_restrict_lex->Height());
-   }
+   localX.SetSize(trialFes->GetNE() * trialFes->GetFE(0)->GetDof() *
+                  trialFes->GetVDim());
+   localY.SetSize(testFes->GetNE() * testFes->GetFE(0)->GetDof() *
+                  testFes->GetVDim());
+   delete elem_restrict;
+   elem_restrict = new ElemRestriction(*fes);
 }
 
 void PABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
@@ -100,52 +97,140 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
+   elem_restrict->Mult(x, localX);
+   localY = 0.0;
    const int iSz = integrators.Size();
-   if (elem_restrict_lex)
+   for (int i = 0; i < iSz; ++i)
    {
-      elem_restrict_lex->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(localX, localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(x, y);
-      }
+      integrators[i]->MultAssembled(localX, localY);
    }
+   elem_restrict->MultTranspose(localY, y);
 }
 
 void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
 {
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   elem_restrict->Mult(x, localX);
+   localY = 0.0;
    const int iSz = integrators.Size();
-   if (elem_restrict_lex)
+   for (int i = 0; i < iSz; ++i)
    {
-      elem_restrict_lex->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposePA(localX, localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
+      integrators[i]->MultAssembledTranspose(localX, localY);
    }
-   else
+   elem_restrict->MultTranspose(localY, y);
+}
+
+
+ElemRestriction::ElemRestriction(const FiniteElementSpace &f)
+   : fes(f),
+     ne(fes.GetNE()),
+     vdim(fes.GetVDim()),
+     byvdim(fes.GetOrdering() == Ordering::byVDIM),
+     ndofs(fes.GetNDofs()),
+     dof(fes.GetFE(0)->GetDof()),
+     nedofs(ne*dof),
+     offsets(ndofs+1),
+     indices(ne*dof)
+{
+   for (int e = 0; e < ne; ++e)
    {
-      y.UseDevice(true);
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      const FiniteElement *fe = fes.GetFE(e);
+      const TensorBasisElement* el =
+         dynamic_cast<const TensorBasisElement*>(fe);
+      if (el) { continue; }
+      mfem_error("Finite element not supported with partial assembly");
+   }
+   const FiniteElement *fe = fes.GetFE(0);
+   const TensorBasisElement* el = dynamic_cast<const TensorBasisElement*>(fe);
+   const Array<int> &dof_map = el->GetDofMap();
+   const bool dof_map_is_identity = (dof_map.Size()==0);
+   const Table& e2dTable = fes.GetElementToDofTable();
+   const int* elementMap = e2dTable.GetJ();
+   // We'll be keeping a count of how many local nodes point to its global dof
+   for (int i = 0; i <= ndofs; ++i)
+   {
+      offsets[i] = 0;
+   }
+   for (int e = 0; e < ne; ++e)
+   {
+      for (int d = 0; d < dof; ++d)
       {
-         integrators[i]->AddMultTransposePA(x, y);
+         const int gid = elementMap[dof*e + d];
+         ++offsets[gid + 1];
       }
    }
+   // Aggregate to find offsets for each global dof
+   for (int i = 1; i <= ndofs; ++i)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+   // For each global dof, fill in all local nodes that point   to it
+   for (int e = 0; e < ne; ++e)
+   {
+      for (int d = 0; d < dof; ++d)
+      {
+         const int did = dof_map_is_identity?d:dof_map[d];
+         const int gid = elementMap[dof*e + did];
+         const int lid = dof*e + d;
+         indices[offsets[gid]++] = lid;
+      }
+   }
+   // We shifted the offsets vector by 1 by using it as a counter
+   // Now we shift it back.
+   for (int i = ndofs; i > 0; --i)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
+}
+
+void ElemRestriction::Mult(const Vector& x, Vector& y) const
+{
+   const int vd = vdim;
+   const bool t = byvdim;
+   const DeviceArray d_offsets(offsets, ndofs+1);
+   const DeviceArray d_indices(indices, nedofs);
+   const DeviceMatrix d_x(x, t?vd:ndofs, t?ndofs:vd);
+   DeviceMatrix d_y(y, t?vd:nedofs, t?nedofs:vd);
+   MFEM_FORALL(i, ndofs,
+   {
+      const int offset = d_offsets[i];
+      const int nextOffset = d_offsets[i+1];
+      for (int c = 0; c < vd; ++c)
+      {
+         const double dofValue = d_x(t?c:i,t?i:c);
+         for (int j = offset; j < nextOffset; ++j)
+         {
+            const int idx_j = d_indices[j];
+            d_y(t?c:idx_j,t?idx_j:c) = dofValue;
+         }
+      }
+   });
+}
+
+void ElemRestriction::MultTranspose(const Vector& x, Vector& y) const
+{
+   const int vd = vdim;
+   const bool t = byvdim;
+   const DeviceArray d_offsets(offsets, ndofs+1);
+   const DeviceArray d_indices(indices, nedofs);
+   const DeviceMatrix d_x(x, t?vd:nedofs, t?nedofs:vd);
+   DeviceMatrix d_y(y, t?vd:ndofs, t?ndofs:vd);
+   MFEM_FORALL(i, ndofs,
+   {
+      const int offset = d_offsets[i];
+      const int nextOffset = d_offsets[i + 1];
+      for (int c = 0; c < vd; ++c)
+      {
+         double dofValue = 0;
+         for (int j = offset; j < nextOffset; ++j)
+         {
+            const int idx_j = d_indices[j];
+            dofValue +=  d_x(t?c:idx_j,t?idx_j:c);
+         }
+         d_y(t?c:i,t?i:c) = dofValue;
+      }
+   });
 }
 
 } // namespace mfem

fem/bilinearform_ext.hpp

@@ -14,16 +14,32 @@
 
 #include "../config/config.hpp"
 #include "fespace.hpp"
-#include "../general/device.hpp"
 
 namespace mfem
 {
 
 class BilinearForm;
 
+/// Element restriction operator
+class ElemRestriction: public Operator
+{
+public:
+   const FiniteElementSpace &fes;
+   const int ne;
+   const int vdim;
+   const bool byvdim;
+   const int ndofs;
+   const int dof;
+   const int nedofs;
+   Array<int> offsets;
+   Array<int> indices;
+public:
+   ElemRestriction(const FiniteElementSpace&);
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
+};
+
 
-/** @brief Class extending the BilinearForm class to support the different
-    AssemblyLevel%s. */
 class BilinearFormExtension : public Operator
 {
 protected:
@@ -32,9 +48,6 @@ protected:
 public:
    BilinearFormExtension(BilinearForm *form);
 
-   virtual MemoryClass GetMemoryClass() const
-   { return Device::GetMemoryClass(); }
-
    /// Get the finite element space prolongation matrix
    virtual const Operator *GetProlongation() const;
 
@@ -67,7 +80,6 @@ public:
                          int copy_interior = 0) {}
    void Mult(const Vector &x, Vector &y) const {}
    void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
    ~FABilinearFormExtension() {}
 };
 
@@ -87,7 +99,6 @@ public:
                          int copy_interior = 0) {}
    void Mult(const Vector &x, Vector &y) const {}
    void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
    ~EABilinearFormExtension() {}
 };
 
@@ -95,9 +106,9 @@ public:
 class PABilinearFormExtension : public BilinearFormExtension
 {
 protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
+   const FiniteElementSpace *trialFes, *testFes;
    mutable Vector localX, localY;
-   const Operator *elem_restrict_lex; // Not owned
+   ElemRestriction *elem_restrict;
 
 public:
    PABilinearFormExtension(BilinearForm*);
@@ -112,6 +123,8 @@ public:
    void Mult(const Vector &x, Vector &y) const;
    void MultTranspose(const Vector &x, Vector &y) const;
    void Update();
+
+   ~PABilinearFormExtension();
 };
 
 /// Data and methods for matrix-free bilinear forms
@@ -130,7 +143,6 @@ public:
                          int copy_interior = 0) {}
    void Mult(const Vector &x, Vector &y) const {}
    void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
    ~MFBilinearFormExtension() {}
 };
 

fem/bilininteg.cpp

@@ -19,20 +19,19 @@ using namespace std;
 
 namespace mfem
 {
-
-void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
+void BilinearFormIntegrator::Assemble(const FiniteElementSpace&)
 {
    mfem_error ("BilinearFormIntegrator::Assemble (...)\n"
                "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
+void BilinearFormIntegrator::MultAssembled(Vector&, Vector&)
 {
    mfem_error ("BilinearFormIntegrator::MultAssembled (...)\n"
                "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
+void BilinearFormIntegrator::MultAssembledTranspose(Vector&, Vector&)
 {
    mfem_error ("BilinearFormIntegrator::MultAssembledTranspose (...)\n"
                "   is not implemented for this class.");
@@ -379,7 +378,6 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
    }
 }
 
-
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -399,7 +397,29 @@ void DiffusionIntegrator::AssembleElementMatrix
 #endif
    elmat.SetSize(nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -455,7 +475,28 @@ void DiffusionIntegrator::AssembleElementMatrix2(
 #endif
    elmat.SetSize(te_nd, tr_nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (trial_fe.Space() == FunctionSpace::Pk)
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
+      }
+      else
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() + dim - 1;
+      }
+
+      if (trial_fe.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      }
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -510,7 +551,29 @@ void DiffusionIntegrator::AssembleElementVector(
 
    elvect.SetSize(nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }
 
    elvect = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -670,27 +733,6 @@ double DiffusionIntegrator::ComputeFluxEnergy
    return energy;
 }
 
-const IntegrationRule &DiffusionIntegrator::GetRule(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe)
-{
-   int order;
-   if (trial_fe.Space() == FunctionSpace::Pk)
-   {
-      order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
-   }
-   else
-   {
-      // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
-      order = trial_fe.GetOrder() + test_fe.GetOrder() + trial_fe.GetDim() - 1;
-   }
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
 
 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
@@ -706,7 +748,21 @@ void MassIntegrator::AssembleElementMatrix
    elmat.SetSize(nd);
    shape.SetSize(nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      // int order = 2 * el.GetOrder();
+      int order = 2 * el.GetOrder() + Trans.OrderW();
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -741,8 +797,13 @@ void MassIntegrator::AssembleElementMatrix2(
    shape.SetSize(tr_nd);
    te_shape.SetSize(te_nd);
 
-   const IntegrationRule *ir = IntRule ? IntRule :
-                               &GetRule(trial_fe, test_fe, Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -763,20 +824,6 @@ void MassIntegrator::AssembleElementMatrix2(
    }
 }
 
-const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
-                                               const FiniteElement &test_fe,
-                                               ElementTransformation &Trans)
-{
-   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
-   const int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
 
 void BoundaryMassIntegrator::AssembleFaceMatrix(
    const FiniteElement &el1, const FiniteElement &el2,
@@ -848,7 +895,7 @@ void ConvectionIntegrator::AssembleElementMatrix(
       ir = &IntRules.Get(el.GetGeomType(), order);
    }
 
-   Q->Eval(Q_ir, Trans, *ir);
+   Q.Eval(Q_ir, Trans, *ir);
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -889,7 +936,7 @@ void GroupConvectionIntegrator::AssembleElementMatrix(
       ir = &IntRules.Get(el.GetGeomType(), order);
    }
 
-   Q->Eval(Q_nodal, Trans, el.GetNodes()); // sets the size of Q_nodal
+   Q.Eval(Q_nodal, Trans, el.GetNodes()); // sets the size of Q_nodal
 
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
@@ -1370,7 +1417,7 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
          dshapedxi(l) = dshapedxt(l,xi);
       }
 
-      shape *= Q->Eval(Trans,ip) * det * ip.weight;
+      shape *= Q.Eval(Trans,ip) * det * ip.weight;
       AddMultVWt (shape, dshapedxi, elmat);
    }
 }
@@ -3216,7 +3263,7 @@ ScalarProductInterpolator::AssembleElementMatrix2(const FiniteElement &dom_fe,
                                                   ElementTransformation &Trans,
                                                   DenseMatrix &elmat)
 {
-   internal::ShapeCoefficient dom_shape_coeff(*Q, dom_fe);
+   internal::ShapeCoefficient dom_shape_coeff(Q, dom_fe);
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 
@@ -3251,7 +3298,7 @@ ScalarVectorProductInterpolator::AssembleElementMatrix2(
       }
    };
 
-   VShapeCoefficient dom_shape_coeff(*Q, dom_fe, Trans.GetSpaceDim());
+   VShapeCoefficient dom_shape_coeff(Q, dom_fe, Trans.GetSpaceDim());
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 
@@ -3289,7 +3336,7 @@ VectorScalarProductInterpolator::AssembleElementMatrix2(
       }
    };
 
-   VecShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   VecShapeCoefficient dom_shape_coeff(VQ, dom_fe);
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 
@@ -3336,11 +3383,11 @@ VectorCrossProductInterpolator::AssembleElementMatrix2(
       }
    };
 
-   VCrossVShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   VCrossVShapeCoefficient dom_shape_coeff(VQ, dom_fe);
 
    if (ran_fe.GetRangeType() == FiniteElement::SCALAR)
    {
-      elmat.SetSize(ran_fe.GetDof()*VQ->GetVDim(),dom_fe.GetDof());
+      elmat.SetSize(ran_fe.GetDof()*VQ.GetVDim(),dom_fe.GetDof());
    }
    else
    {
@@ -3389,7 +3436,7 @@ VectorInnerProductInterpolator::AssembleElementMatrix2(
    ElementTransformation &Trans,
    DenseMatrix &elmat)
 {
-   internal::VDotVShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   internal::VDotVShapeCoefficient dom_shape_coeff(VQ, dom_fe);
 
    elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());
 

fem/bilininteg.hpp

@@ -15,6 +15,7 @@
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
 #include "fespace.hpp"
+#include "bilininteg_ext.hpp"
 
 namespace mfem
 {
@@ -22,45 +23,19 @@ namespace mfem
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
-protected:
-   BilinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : NonlinearFormIntegrator(ir) { }
+public:
+   BilinearFormIntegrator(const IntegrationRule *ir = NULL) :
+      NonlinearFormIntegrator(ir) { }
 
 public:
-   // TODO: add support for other assembly levels (in addition to PA) and their
-   // actions.
-
-   // TODO: for mixed meshes the quadrature rules to be used by methods like
-   // AssemblePA() can be given as a QuadratureSpace, e.g. using a new method:
-   // SetQuadratureSpace().
-
-   // TODO: the methods for the various assembly levels make sense even in the
-   // base class NonlinearFormIntegrator, except that not all assembly levels
-   // make sense for the action of the nonlinear operator (but they all make
-   // sense for its Jacobian).
-
    /// Method defining partial assembly.
-   /** The result of the partial assembly is stored internally so that it can be
-       used later in the methods AddMultPA() and AddMultTransposePA(). */
-   virtual void AssemblePA(const FiniteElementSpace &fes);
+   virtual void Assemble(const FiniteElementSpace&);
 
    /// Method for partially assembled action.
-   /** Perform the action of integrator on the input @a x and add the result to
-       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
-       the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void MultAssembled(Vector&, Vector&);
 
    /// Method for partially assembled transposed action.
-   /** Perform the transpose action of integrator on the input @a x and add the
-       result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
-       represent the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+   virtual void MultAssembledTranspose(Vector&, Vector&);
 
    /// Given a particular Finite Element computes the element matrix elmat.
    virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -309,10 +284,10 @@ protected:
                                       Vector & shape)
    { trial_fe.CalcPhysShape(Trans, shape); }
 
-   Coefficient *Q;
-
 private:
 
+   Coefficient *Q;
+
 #ifndef MFEM_THREAD_SAFE
    Vector test_shape;
    Vector trial_shape;
@@ -383,13 +358,13 @@ protected:
                                       DenseMatrix & shape)
    { trial_fe.CalcVShape(Trans, shape); }
 
+private:
+
    Coefficient *Q;
    VectorCoefficient *VQ;
    VectorCoefficient *DQ;
    MatrixCoefficient *MQ;
 
-private:
-
 #ifndef MFEM_THREAD_SAFE
    Vector V;
    Vector D;
@@ -464,12 +439,12 @@ protected:
                                  Vector & shape)
    { scalar_fe.CalcPhysShape(Trans, shape); }
 
+private:
+
    VectorCoefficient *VQ;
    bool transpose;
    bool cross_2d;  // In 2D use a cross product rather than a dot product
 
-private:
-
 #ifndef MFEM_THREAD_SAFE
    Vector V;
    DenseMatrix vshape;
@@ -1662,34 +1637,27 @@ protected:
     can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-   MatrixCoefficient *MQ;
-
 private:
    Vector vec, pointflux, shape;
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape, dshapedxt, invdfdx, mq;
    DenseMatrix te_dshape, te_dshapedxt;
 #endif
-
+   Coefficient *Q;
+   MatrixCoefficient *MQ;
    // PA extension
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
+   DofToQuad *maps;
+   GeometryExtension *geom;
    int dim, ne, dofs1D, quad1D;
-   Vector pa_data;
-
 public:
    /// Construct a diffusion integrator with coefficient Q = 1
    DiffusionIntegrator() { Q = NULL; MQ = NULL; maps = NULL; geom = NULL; }
 
    /// Construct a diffusion integrator with a scalar coefficient q
-   DiffusionIntegrator(Coefficient &q)
-      : Q(&q) { MQ = NULL; maps = NULL; geom = NULL; }
+   DiffusionIntegrator (Coefficient &q) : Q(&q) { MQ = NULL; maps = NULL; geom = NULL; }
 
    /// Construct a diffusion integrator with a matrix coefficient q
-   DiffusionIntegrator(MatrixCoefficient &q)
-      : MQ(&q) { Q = NULL; maps = NULL; geom = NULL; }
+   DiffusionIntegrator (MatrixCoefficient &q) : MQ(&q) { Q = NULL; maps = NULL; geom = NULL; }
 
    /** Given a particular Finite Element
        computes the element stiffness matrix elmat. */
@@ -1717,12 +1685,11 @@ public:
                                     ElementTransformation &Trans,
                                     Vector &flux, Vector *d_energy = NULL);
 
-   virtual void AssemblePA(const FiniteElementSpace&);
+   /// PA extension
+   virtual void Assemble(const FiniteElementSpace&);
+   virtual void MultAssembled(Vector&, Vector&);
 
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe);
+   virtual ~DiffusionIntegrator();
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1734,15 +1701,13 @@ protected:
 #endif
    Coefficient *Q;
    // PA extension
-   Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
+   Vector vec;
+   DofToQuad *maps;
+   GeometryExtension *geom;
    int dim, ne, nq, dofs1D, quad1D;
-
 public:
    MassIntegrator(const IntegrationRule *ir = NULL)
       : BilinearFormIntegrator(ir) { Q = NULL; maps = NULL; geom = NULL; }
-
    /// Construct a mass integrator with coefficient q
    MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
       : BilinearFormIntegrator(ir), Q(&q) { maps = NULL; geom = NULL; }
@@ -1756,14 +1721,11 @@ public:
                                        const FiniteElement &test_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
+   /// PA extension
+   virtual void Assemble(const FiniteElementSpace&);
+   virtual void MultAssembled(Vector&, Vector&);
 
-   virtual void AssemblePA(const FiniteElementSpace&);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
+   virtual ~MassIntegrator();
 };
 
 class BoundaryMassIntegrator : public MassIntegrator
@@ -1782,19 +1744,17 @@ public:
 /// alpha (q . grad u, v)
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
-protected:
-   VectorCoefficient *Q;
-   double alpha;
-
 private:
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape, adjJ, Q_ir;
    Vector shape, vec2, BdFidxT;
 #endif
+   VectorCoefficient &Q;
+   double alpha;
 
 public:
    ConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
-      : Q(&q) { alpha = a; }
+      : Q(q) { alpha = a; }
    virtual void AssembleElementMatrix(const FiniteElement &,
                                       ElementTransformation &,
                                       DenseMatrix &);
@@ -1803,17 +1763,15 @@ public:
 /// alpha (q . grad u, v) using the "group" FE discretization
 class GroupConvectionIntegrator : public BilinearFormIntegrator
 {
-protected:
-   VectorCoefficient *Q;
-   double alpha;
-
 private:
    DenseMatrix dshape, adjJ, Q_nodal, grad;
    Vector shape;
+   VectorCoefficient &Q;
+   double alpha;
 
 public:
    GroupConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
-      : Q(&q) { alpha = a; }
+      : Q(q) { alpha = a; }
    virtual void AssembleElementMatrix(const FiniteElement &,
                                       ElementTransformation &,
                                       DenseMatrix &);
@@ -1829,17 +1787,16 @@ private:
    Vector shape, te_shape, vec;
    DenseMatrix partelmat;
    DenseMatrix mcoeff;
-   int Q_order;
-
-protected:
    Coefficient *Q;
    VectorCoefficient *VQ;
    MatrixCoefficient *MQ;
 
+   int Q_order;
+
 public:
    /// Construct an integrator with coefficient 1.0
    VectorMassIntegrator()
-      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
+      : vdim(-1), Q(NULL), VQ(NULL), MQ(NULL), Q_order(0) { }
    /** Construct an integrator with scalar coefficient q.
        If possible, save memory by using a scalar integrator since
        the resulting matrix is block diagonal with the same diagonal
@@ -1878,14 +1835,11 @@ public:
     does NOT depend on the ElementTransformation Trans. */
 class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
    Vector divshape, shape;
 #endif
-
 public:
    VectorFEDivergenceIntegrator() { Q = NULL; }
    VectorFEDivergenceIntegrator(Coefficient &q) { Q = &q; }
@@ -1903,17 +1857,14 @@ public:
     This is equivalent to a weak divergence of the Nedelec basis functions. */
 class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape;
    DenseMatrix dshapedxt;
    DenseMatrix vshape;
    DenseMatrix invdfdx;
 #endif
-
 public:
    VectorFEWeakDivergenceIntegrator() { Q = NULL; }
    VectorFEWeakDivergenceIntegrator(Coefficient &q) { Q = &q; }
@@ -1930,16 +1881,13 @@ public:
     test spaces are switched, assembles the form (u, curl v). */
 class VectorFECurlIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix curlshapeTrial;
    DenseMatrix vshapeTest;
    DenseMatrix curlshapeTrial_dFT;
 #endif
-
 public:
    VectorFECurlIntegrator() { Q = NULL; }
    VectorFECurlIntegrator(Coefficient &q) { Q = &q; }
@@ -1952,19 +1900,17 @@ public:
                                        DenseMatrix &elmat);
 };
 
+
 /// Class for integrating (Q D_i(u), v); u and v are scalars
 class DerivativeIntegrator : public BilinearFormIntegrator
 {
-protected:
-   Coefficient* Q;
-
 private:
+   Coefficient & Q;
    int xi;
    DenseMatrix dshape, dshapedxt, invdfdx;
    Vector shape, dshapedxi;
-
 public:
-   DerivativeIntegrator(Coefficient &q, int i) : Q(&q), xi(i) { }
+   DerivativeIntegrator(Coefficient &q, int i) : Q(q), xi(i) { }
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat)
@@ -1984,8 +1930,6 @@ private:
    DenseMatrix curlshape, curlshape_dFt, M;
    DenseMatrix vshape, projcurl;
 #endif
-
-protected:
    Coefficient *Q;
    MatrixCoefficient *MQ;
 
@@ -2019,8 +1963,6 @@ private:
 #ifndef MFEM_THREAD_SAFE
    DenseMatrix dshape_hat, dshape, curlshape, Jadj, grad_hat, grad;
 #endif
-
-protected:
    Coefficient *Q;
 
 public:
@@ -2042,6 +1984,9 @@ public:
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
+   Coefficient *Q;
+   VectorCoefficient *VQ;
+   MatrixCoefficient *MQ;
    void Init(Coefficient *q, VectorCoefficient *vq, MatrixCoefficient *mq)
    { Q = q; VQ = vq; MQ = mq; }
 
@@ -2053,11 +1998,6 @@ private:
    DenseMatrix trial_vshape;
 #endif
 
-protected:
-   Coefficient *Q;
-   VectorCoefficient *VQ;
-   MatrixCoefficient *MQ;
-
 public:
    VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
    VectorFEMassIntegrator(Coefficient *_q) { Init(_q, NULL, NULL); }
@@ -2080,10 +2020,9 @@ public:
     scalar FE space; p is also in a (different) scalar FE space.  */
 class VectorDivergenceIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *Q;
 
-private:
    Vector shape;
    Vector divshape;
    DenseMatrix dshape;
@@ -2104,10 +2043,9 @@ public:
 /// (Q div u, div v) for RT elements
 class DivDivIntegrator: public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *Q;
 
-private:
 #ifndef MFEM_THREAD_SAFE
    Vector divshape;
 #endif
@@ -2129,10 +2067,9 @@ public:
     diffusion matrix in each diagonal block. */
 class VectorDiffusionIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *Q;
 
-private:
    DenseMatrix Jinv;
    DenseMatrix dshape;
    DenseMatrix gshape;
@@ -2157,11 +2094,10 @@ public:
     using multiple copies of a scalar FE space. */
 class ElasticityIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    double q_lambda, q_mu;
    Coefficient *lambda, *mu;
 
-private:
 #ifndef MFEM_THREAD_SAFE
    Vector shape;
    DenseMatrix dshape, gshape, pelmat;
@@ -2218,12 +2154,11 @@ public:
     points. */
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
    Coefficient *rho;
    VectorCoefficient *u;
    double alpha, beta;
 
-private:
    Vector shape1, shape2;
 
 public:
@@ -2510,7 +2445,7 @@ public:
 class ScalarProductInterpolator : public DiscreteInterpolator
 {
 public:
-   ScalarProductInterpolator(Coefficient & sc) : Q(&sc) { }
+   ScalarProductInterpolator(Coefficient & sc) : Q(sc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
@@ -2518,7 +2453,7 @@ public:
                                        DenseMatrix &elmat);
 
 protected:
-   Coefficient *Q;
+   Coefficient &Q;
 };
 
 /** Interpolator of a scalar coefficient multiplied by a vector field onto
@@ -2528,14 +2463,14 @@ class ScalarVectorProductInterpolator : public DiscreteInterpolator
 {
 public:
    ScalarVectorProductInterpolator(Coefficient & sc)
-      : Q(&sc) { }
+      : Q(sc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   Coefficient *Q;
+   Coefficient &Q;
 };
 
 /** Interpolator of a vector coefficient multiplied by a scalar field onto
@@ -2545,14 +2480,14 @@ class VectorScalarProductInterpolator : public DiscreteInterpolator
 {
 public:
    VectorScalarProductInterpolator(VectorCoefficient & vc)
-      : VQ(&vc) { }
+      : VQ(vc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };
 
 /** Interpolator of the cross product between a vector coefficient and an
@@ -2562,14 +2497,14 @@ class VectorCrossProductInterpolator : public DiscreteInterpolator
 {
 public:
    VectorCrossProductInterpolator(VectorCoefficient & vc)
-      : VQ(&vc) { }
+      : VQ(vc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &nd_fe,
                                        const FiniteElement &rt_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };
 
 /** Interpolator of the inner product between a vector coefficient and an
@@ -2578,14 +2513,14 @@ protected:
 class VectorInnerProductInterpolator : public DiscreteInterpolator
 {
 public:
-   VectorInnerProductInterpolator(VectorCoefficient & vc) : VQ(&vc) { }
+   VectorInnerProductInterpolator(VectorCoefficient & vc) : VQ(vc) { }
 
    virtual void AssembleElementMatrix2(const FiniteElement &rt_fe,
                                        const FiniteElement &l2_fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };
 
 }

fem/bilininteg_ext.hpp

@@ -0,0 +1,80 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BILININTEG_EXT
+#define MFEM_BILININTEG_EXT
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+/// GeometryExtension
+class GeometryExtension
+{
+public:
+   Array<int> eMap;
+   Array<double> nodes;
+   Array<double> X, J, invJ, detJ;
+   static GeometryExtension* Get(const FiniteElementSpace&,
+                                 const IntegrationRule&);
+   static GeometryExtension* Get(const FiniteElementSpace&,
+                                 const IntegrationRule&,
+                                 const Vector&);
+   static void ReorderByVDim(const GridFunction*);
+   static void ReorderByNodes(const GridFunction*);
+};
+
+/// DofToQuad
+class DofToQuad
+{
+private:
+   std::string hash;
+public:
+   ~DofToQuad();
+   void operator=(DofToQuad&);
+   void operator=(DofToQuad const&);
+public:
+   Array<double> W, B, G, Bt, Gt;
+public:
+   static DofToQuad* Get(const FiniteElementSpace&,
+                         const IntegrationRule&,
+                         const bool = false);
+   static DofToQuad* Get(const FiniteElementSpace&,
+                         const FiniteElementSpace&,
+                         const IntegrationRule&,
+                         const bool = false);
+   static DofToQuad* Get(const FiniteElement&,
+                         const FiniteElement&,
+                         const IntegrationRule&,
+                         const bool = false);
+   static DofToQuad* GetTensorMaps(const FiniteElement&,
+                                   const FiniteElement&,
+                                   const IntegrationRule&,
+                                   const bool = false);
+   static DofToQuad* GetD2QTensorMaps(const FiniteElement&,
+                                      const IntegrationRule&,
+                                      const bool = false);
+   static DofToQuad* GetSimplexMaps(const FiniteElement&,
+                                    const IntegrationRule&,
+                                    const bool = false);
+   static DofToQuad* GetSimplexMaps(const FiniteElement&,
+                                    const FiniteElement&,
+                                    const IntegrationRule&,
+                                    const bool = false);
+   static DofToQuad* GetD2QSimplexMaps(const FiniteElement&,
+                                       const IntegrationRule&,
+                                       const bool = false);
+};
+
+}
+
+#endif

fem/bilininteg_mass.cpp

@@ -1,822 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Mass Integrator
-
-// PA Mass Assemble kernel
-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   // Assuming the same element type
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
-   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
-                                    GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, Device::GetMemoryType());
-   ConstantCoefficient *const_coeff = dynamic_cast<ConstantCoefficient*>(Q);
-   // TODO: other types of coefficients ...
-   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
-   if (dim==2)
-   {
-      double constant = 0.0;
-      if (const_coeff)
-      {
-         constant = const_coeff->constant;
-      }
-      else
-      {
-         MFEM_ABORT("Coefficient type not supported");
-      }
-      const int NE = ne;
-      const int NQ = nq;
-      auto w = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      MFEM_FORALL(e, NE,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double J11 = J(q,0,0,e);
-            const double J12 = J(q,1,0,e);
-            const double J21 = J(q,0,1,e);
-            const double J22 = J(q,1,1,e);
-            const double detJ = (J11*J22)-(J21*J12);
-            v(q,e) =  w[q] * constant * detJ;
-         }
-      });
-   }
-   if (dim==3)
-   {
-      double constant = 0.0;
-      if (const_coeff)
-      {
-         constant = const_coeff->constant;
-      }
-      else
-      {
-         MFEM_ABORT("Coefficient type not supported");
-      }
-      const int NE = ne;
-      const int NQ = nq;
-      auto W = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
-      auto v = Reshape(pa_data.Write(), NQ,NE);
-      MFEM_FORALL(e, NE,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e);
-            const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e);
-            const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e);
-            const double detJ = J11 * (J22 * J33 - J32 * J23) -
-            /* */               J21 * (J12 * J33 - J32 * J13) +
-            /* */               J31 * (J12 * J23 - J22 * J13);
-            v(q,e) = W[q] * constant * detJ;
-         }
-      });
-   }
-}
-
-#ifdef MFEM_USE_OCCA
-// OCCA PA Mass Apply 2D kernel
-static void OccaPAMassApply2D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &op,
-                              const Vector &x,
-                              Vector &y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_op = OccaMemoryRead(op.GetMemory(), op.Size());
-   const occa::memory o_x = OccaMemoryRead(x.GetMemory(), x.Size());
-   occa::memory o_y = OccaMemoryReadWrite(y.GetMemory(), y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply2D_cpu;
-      if (OccaMassApply2D_cpu.find(id) == OccaMassApply2D_cpu.end())
-      {
-         const occa::kernel MassApply2D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_CPU", props);
-         OccaMassApply2D_cpu.emplace(id, MassApply2D_CPU);
-      }
-      OccaMassApply2D_cpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply2D_gpu;
-      if (OccaMassApply2D_gpu.find(id) == OccaMassApply2D_gpu.end())
-      {
-         const occa::kernel MassApply2D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_GPU", props);
-         OccaMassApply2D_gpu.emplace(id, MassApply2D_GPU);
-      }
-      OccaMassApply2D_gpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-}
-
-// OCCA PA Mass Apply 3D kernel
-static void OccaPAMassApply3D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &op,
-                              const Vector &x,
-                              Vector &y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_op = OccaMemoryRead(op.GetMemory(), op.Size());
-   const occa::memory o_x = OccaMemoryRead(x.GetMemory(), x.Size());
-   occa::memory o_y = OccaMemoryReadWrite(y.GetMemory(), y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply3D_cpu;
-      if (OccaMassApply3D_cpu.find(id) == OccaMassApply3D_cpu.end())
-      {
-         const occa::kernel MassApply3D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_CPU", props);
-         OccaMassApply3D_cpu.emplace(id, MassApply3D_CPU);
-      }
-      OccaMassApply3D_cpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply3D_gpu;
-      if (OccaMassApply3D_gpu.find(id) == OccaMassApply3D_gpu.end())
-      {
-         const occa::kernel MassApply3D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_GPU", props);
-         OccaMassApply3D_gpu.emplace(id, MassApply3D_GPU);
-      }
-      OccaMassApply3D_gpu.at(id)(NE, o_B, o_Bt, o_op, o_x, o_y);
-   }
-}
-#endif // MFEM_USE_OCCA
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void PAMassApply2D(const int NE,
-                          const Array<double> &B_,
-                          const Array<double> &Bt_,
-                          const Vector &op_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(B_.Read(), Q1D, D1D);
-   auto Bt = Reshape(Bt_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         double sol_x[max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            sol_x[qy] = 0.0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double s = x(dx,dy,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] += B(qx,dx)* s;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double d2q = B(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] += d2q * sol_x[qx];
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] *= op(qx,qy,e);
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double sol_x[max_D1D];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] = 0.0;
-         }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double s = sol_xy[qy][qx];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] += Bt(dx,qx) * s;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            const double q2d = Bt(dy,qy);
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               y(dx,dy,e) += q2d * sol_x[dx];
-            }
-         }
-      }
-   });
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0,
-         const int T_NBZ = 0>
-static void SmemPAMassApply2D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double BBt[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) BBt;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
-      MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
-      double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
-      double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
-      double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
-      double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            X[dy][dx] = x(dx,dy,e);
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double dq = 0.0;
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               dq += X[dy][dx] * B[qx][dx];
-            }
-            DQ[dy][qx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double qq = 0.0;
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               qq += DQ[dy][qx] * B[qy][dy];
-            }
-            QQ[qy][qx] = qq * op(qx, qy, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[d][q] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dq = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dq += QQ[qy][qx] * Bt[dx][qx];
-            }
-            QD[qy][dx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dd = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               dd += (QD[qy][dx] * Bt[dy][qy]);
-            }
-            y(dx, dy, e) += dd;
-         }
-      }
-   });
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void PAMassApply3D(const int NE,
-                          const Array<double> &B_,
-                          const Array<double> &Bt_,
-                          const Vector &op_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(B_.Read(), Q1D, D1D);
-   auto Bt = Reshape(Bt_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         double sol_xy[max_Q1D][max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double sol_x[max_Q1D];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] = 0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double s = x(dx,dy,dz,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_x[qx] += B(qx,dx) * s;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy = B(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xy[qy][qx] += wy * sol_x[qx];
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const double wz = B(qz,dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-               }
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] *= op(qx,qy,qz,e);
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         double sol_xy[max_D1D][max_D1D];
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_xy[dy][dx] = 0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double sol_x[max_D1D];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] = 0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double s = sol_xyz[qz][qy][qx];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_x[dx] += Bt(dx,qx) * s;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double wy = Bt(dy,qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_xy[dy][dx] += wy * sol_x[dx];
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            const double wz = Bt(dz,qz);
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
-               }
-            }
-         }
-      }
-   });
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void SmemPAMassApply3D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= M1D, "");
-   MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sDQ[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sDQ;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
-      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-      double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
-      double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
-      double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
-      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
-      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
-      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               X[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  u += X[dz][dy][dx] * B[qx][dx];
-               }
-               DDQ[dz][dy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-               DQQ[dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u += DQQ[dz][qy][qx] * B[qz][dz];
-               }
-               QQQ[qz][qy][qx] = u * op(qx,qy,qz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[d][q] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += QQQ[qz][qy][qx] * Bt[dx][qx];
-               }
-               QQD[qz][qy][dx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-               QDD[qz][dy][dx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-               y(dx,dy,dz,e) += u;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassApply(const int dim,
-                        const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const Array<double> &B,
-                        const Array<double> &Bt,
-                        const Vector &op,
-                        const Vector &x,
-                        Vector &y)
-{
-#ifdef MFEM_USE_OCCA
-   if (DeviceCanUseOcca())
-   {
-      if (dim == 2)
-      {
-         OccaPAMassApply2D(D1D, Q1D, NE, B, Bt, op, x, y);
-         return;
-      }
-      if (dim == 3)
-      {
-         OccaPAMassApply3D(D1D, Q1D, NE, B, Bt, op, x, y);
-         return;
-      }
-      MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
-   }
-#endif // MFEM_USE_OCCA
-
-   if (Device::Allows(Backend::RAJA_CUDA))
-   {
-      if (dim == 2)
-      {
-         switch ((D1D << 4 ) | Q1D)
-         {
-            case 0x22: return PAMassApply2D<2,2>(NE, B, Bt, op, x, y);
-            case 0x33: return PAMassApply2D<3,3>(NE, B, Bt, op, x, y);
-            case 0x44: return PAMassApply2D<4,4>(NE, B, Bt, op, x, y);
-            case 0x55: return PAMassApply2D<5,5>(NE, B, Bt, op, x, y);
-            case 0x66: return PAMassApply2D<6,6>(NE, B, Bt, op, x, y);
-            case 0x77: return PAMassApply2D<7,7>(NE, B, Bt, op, x, y);
-            case 0x88: return PAMassApply2D<8,8>(NE, B, Bt, op, x, y);
-            case 0x99: return PAMassApply2D<9,9>(NE, B, Bt, op, x, y);
-            default:   return PAMassApply2D(NE, B, Bt, op, x, y, D1D, Q1D);
-         }
-      }
-      if (dim == 3)
-      {
-         switch ((D1D << 4 ) | Q1D)
-         {
-            case 0x23: return PAMassApply3D<2,3>(NE, B, Bt, op, x, y);
-            case 0x34: return PAMassApply3D<3,4>(NE, B, Bt, op, x, y);
-            case 0x45: return PAMassApply3D<4,5>(NE, B, Bt, op, x, y);
-            case 0x56: return PAMassApply3D<5,6>(NE, B, Bt, op, x, y);
-            case 0x67: return PAMassApply3D<6,7>(NE, B, Bt, op, x, y);
-            case 0x78: return PAMassApply3D<7,8>(NE, B, Bt, op, x, y);
-            case 0x89: return PAMassApply3D<8,9>(NE, B, Bt, op, x, y);
-            default:   return PAMassApply3D(NE, B, Bt, op, x, y, D1D, Q1D);
-         }
-      }
-   }
-   else if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAMassApply2D<2,2,16>(NE, B, Bt, op, x, y);
-         case 0x33: return SmemPAMassApply2D<3,3,16>(NE, B, Bt, op, x, y);
-         case 0x44: return SmemPAMassApply2D<4,4,8>(NE, B, Bt, op, x, y);
-         case 0x55: return SmemPAMassApply2D<5,5,8>(NE, B, Bt, op, x, y);
-         case 0x66: return SmemPAMassApply2D<6,6,4>(NE, B, Bt, op, x, y);
-         case 0x77: return SmemPAMassApply2D<7,7,4>(NE, B, Bt, op, x, y);
-         case 0x88: return SmemPAMassApply2D<8,8,2>(NE, B, Bt, op, x, y);
-         case 0x99: return SmemPAMassApply2D<9,9,2>(NE, B, Bt, op, x, y);
-         default:   return PAMassApply2D(NE, B, Bt, op, x, y, D1D, Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAMassApply3D<2,3>(NE, B, Bt, op, x, y);
-         case 0x34: return SmemPAMassApply3D<3,4>(NE, B, Bt, op, x, y);
-         case 0x45: return SmemPAMassApply3D<4,5>(NE, B, Bt, op, x, y);
-         case 0x56: return SmemPAMassApply3D<5,6>(NE, B, Bt, op, x, y);
-         case 0x67: return SmemPAMassApply3D<6,7>(NE, B, Bt, op, x, y);
-         case 0x78: return SmemPAMassApply3D<7,8>(NE, B, Bt, op, x, y);
-         case 0x89: return SmemPAMassApply3D<8,9>(NE, B, Bt, op, x, y);
-         default:   return PAMassApply3D(NE, B, Bt, op, x, y, D1D, Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
-}
-
-} // namespace mfem

fem/coefficient.cpp

@@ -28,6 +28,11 @@ double PWConstCoefficient::Eval(ElementTransformation & T,
    return (constants(att-1));
 }
 
+DeviceFunctionCoefficientPtr FunctionCoefficient::GetDeviceFunction()
+{
+   return DeviceFunction;
+}
+
 double FunctionCoefficient::Eval(ElementTransformation & T,
                                  const IntegrationPoint & ip)
 {
@@ -40,6 +45,10 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
    {
       return ((*Function)(transip));
    }
+   else if (DeviceFunction)
+   {
+      return ((*DeviceFunction)(Vector3(x)));
+   }
    else
    {
       return (*TDFunction)(transip, GetTime());
@@ -125,27 +134,19 @@ void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
 }
 
 VectorArrayCoefficient::VectorArrayCoefficient (int dim)
-   : VectorCoefficient(dim), Coeff(dim), ownCoeff(dim)
+   : VectorCoefficient(dim), Coeff(dim)
 {
    for (int i = 0; i < dim; i++)
    {
       Coeff[i] = NULL;
-      ownCoeff[i] = true;
    }
 }
 
-void VectorArrayCoefficient::Set(int i, Coefficient *c, bool own)
-{
-   if (ownCoeff[i]) { delete Coeff[i]; }
-   Coeff[i] = c;
-   ownCoeff[i] = own;
-}
-
 VectorArrayCoefficient::~VectorArrayCoefficient()
 {
    for (int i = 0; i < vdim; i++)
    {
-      if (ownCoeff[i]) { delete Coeff[i]; }
+      delete Coeff[i];
    }
 }
 
@@ -317,26 +318,17 @@ MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
    : MatrixCoefficient (dim)
 {
    Coeff.SetSize(height*width);
-   ownCoeff.SetSize(height*width);
    for (int i = 0; i < (height*width); i++)
    {
       Coeff[i] = NULL;
-      ownCoeff[i] = true;
    }
 }
 
-void MatrixArrayCoefficient::Set(int i, int j, Coefficient * c, bool own)
-{
-   if (ownCoeff[i*width+j]) { delete Coeff[i*width+j]; }
-   Coeff[i*width+j] = c;
-   ownCoeff[i*width+j] = own;
-}
-
 MatrixArrayCoefficient::~MatrixArrayCoefficient ()
 {
    for (int i=0; i < height*width; i++)
    {
-      if (ownCoeff[i]) { delete Coeff[i]; }
+      delete Coeff[i];
    }
 }
 

fem/coefficient.hpp

@@ -112,6 +112,7 @@ public:
                        const IntegrationPoint &ip);
 };
 
+typedef double (*DeviceFunctionCoefficientPtr)(const Vector3&);
 
 /// class for C-function coefficient
 class FunctionCoefficient : public Coefficient
@@ -119,6 +120,7 @@ class FunctionCoefficient : public Coefficient
 protected:
    double (*Function)(const Vector &);
    double (*TDFunction)(const Vector &, double);
+   double (*DeviceFunction)(const Vector3&);
 
 public:
    /// Define a time-independent coefficient from a C-function
@@ -126,6 +128,7 @@ public:
    {
       Function = f;
       TDFunction = NULL;
+      DeviceFunction = NULL;
    }
 
    /// Define a time-dependent coefficient from a C-function
@@ -133,6 +136,16 @@ public:
    {
       Function = NULL;
       TDFunction = tdf;
+      DeviceFunction = NULL;
+   }
+
+   /// Define a time-independent coefficient from a C-function using
+   /// Vector3 instead of a Vector.
+   FunctionCoefficient(double (*df)(const Vector3 &))
+   {
+      Function = NULL;
+      TDFunction = NULL;
+      DeviceFunction = df;
    }
 
    /// (DEPRECATED) Define a time-independent coefficient from a C-function
@@ -142,6 +155,7 @@ public:
    {
       Function = reinterpret_cast<double(*)(const Vector&)>(f);
       TDFunction = NULL;
+      DeviceFunction = NULL;
    }
 
    /// (DEPRECATED) Define a time-dependent coefficient from a C-function
@@ -151,11 +165,17 @@ public:
    {
       Function = NULL;
       TDFunction = reinterpret_cast<double(*)(const Vector&,double)>(tdf);
+      DeviceFunction = NULL;
    }
 
    /// Evaluate coefficient
    virtual double Eval(ElementTransformation &T,
                        const IntegrationPoint &ip);
+
+   /// Return the coefficient's C-function that uses Vector3.
+   /// Warning: for now, the returned function can only be used on the
+   /// host inside a MFEM_FORALL.
+   DeviceFunctionCoefficientPtr GetDeviceFunction();
 };
 
 class GridFunction;
@@ -369,7 +389,6 @@ class VectorArrayCoefficient : public VectorCoefficient
 {
 private:
    Array<Coefficient*> Coeff;
-   Array<bool> ownCoeff;
 
 public:
    /// Construct vector of dim coefficients.
@@ -381,7 +400,7 @@ public:
    Coefficient **GetCoeffs() { return Coeff; }
 
    /// Sets coefficient in the vector.
-   void Set(int i, Coefficient *c, bool own=true);
+   void Set(int i, Coefficient *c) { delete Coeff[i]; Coeff[i] = c; }
 
    /// Evaluates i'th component of the vector.
    double Eval(int i, ElementTransformation &T, const IntegrationPoint &ip)
@@ -501,13 +520,9 @@ public:
    void SetDeltaCoefficient(const DeltaCoefficient& _d) { d = _d; }
    /// Return the associated scalar DeltaCoefficient.
    DeltaCoefficient& GetDeltaCoefficient() { return d; }
-
-   void SetScale(double s) { d.SetScale(s); }
    void SetDirection(const Vector& _d);
 
-   void SetDeltaCenter(const Vector& center) { d.SetDeltaCenter(center); }
    void GetDeltaCenter(Vector& center) { d.GetDeltaCenter(center); }
-
    /** @brief Return the specified direction vector multiplied by the value
        returned by DeltaCoefficient::EvalDelta() of the associated scalar
        DeltaCoefficient. */
@@ -633,7 +648,6 @@ class MatrixArrayCoefficient : public MatrixCoefficient
 {
 private:
    Array<Coefficient *> Coeff;
-   Array<bool> ownCoeff;
 
 public:
 
@@ -641,7 +655,7 @@ public:
 
    Coefficient* GetCoeff (int i, int j) { return Coeff[i*width+j]; }
 
-   void Set(int i, int j, Coefficient * c, bool own=true);
+   void Set(int i, int j, Coefficient * c) { delete Coeff[i*width+j]; Coeff[i*width+j] = c; }
 
    double Eval(int i, int j, ElementTransformation &T, const IntegrationPoint &ip)
    { return Coeff[i*width+j] ? Coeff[i*width+j] -> Eval(T, ip, GetTime()) : 0.0; }

fem/complex_fem.cpp

@@ -0,0 +1,784 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#include "complex_fem.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
+   : Vector(2*(fes->GetVSize()))
+{
+   gfr_ = new GridFunction(fes, &data[0]);
+   gfi_ = new GridFunction(fes, &data[fes->GetVSize()]);
+}
+
+void
+ComplexGridFunction::Update()
+{
+   FiniteElementSpace * fes = gfr_->FESpace();
+
+   int vsize = fes->GetVSize();
+
+   const Operator *T = fes->GetUpdateOperator();
+   if (T)
+   {
+      // Update the individual GridFunction objects.  This will allocate
+      // new data arrays for each GridFunction.
+      gfr_->Update();
+      gfi_->Update();
+
+      // Our data array now contains old data as well as being the wrong size
+      // so reallocate it.
+      this->SetSize(2 * vsize);
+
+      // Create temporary vectors which point to the new data array
+      Vector gf_r(&data[0], vsize);
+      Vector gf_i(&data[vsize], vsize);
+
+      // Copy the updated GridFunctions into the new data array
+      gf_r = *gfr_;
+      gf_i = *gfi_;
+
+      // Replace the individual data arrays with pointers into the new data array
+      gfr_->NewDataAndSize(&data[0], vsize);
+      gfi_->NewDataAndSize(&data[vsize], vsize);
+   }
+   else
+   {
+      // The existing data will not be transferred to the new GridFunctions
+      // so delete it a allocate a new array
+      this->SetSize(2 * vsize);
+
+      // Point the individual GridFunctions to the new data array
+      gfr_->NewDataAndSize(&data[0], vsize);
+      gfi_->NewDataAndSize(&data[vsize], vsize);
+
+      // These updates will only set the proper 'sequence' value within
+      // the individual GridFunction objects because their sizes are
+      // already correct
+      gfr_->Update();
+      gfi_->Update();
+   }
+}
+
+void
+ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
+                                        Coefficient &imag_coeff)
+{
+   gfr_->ProjectCoefficient(real_coeff);
+   gfi_->ProjectCoefficient(imag_coeff);
+}
+
+void
+ComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
+                                        VectorCoefficient &imag_vcoeff)
+{
+   gfr_->ProjectCoefficient(real_vcoeff);
+   gfi_->ProjectCoefficient(imag_vcoeff);
+}
+
+void
+ComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
+                                           Coefficient &imag_coeff,
+                                           Array<int> &attr)
+{
+   gfr_->ProjectBdrCoefficient(real_coeff, attr);
+   gfi_->ProjectBdrCoefficient(imag_coeff, attr);
+}
+
+void
+ComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient &real_vcoeff,
+                                                 VectorCoefficient &imag_vcoeff,
+                                                 Array<int> &attr)
+{
+   gfr_->ProjectBdrCoefficientNormal(real_vcoeff, attr);
+   gfi_->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
+}
+
+void
+ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
+                                                  &real_vcoeff,
+                                                  VectorCoefficient
+                                                  &imag_vcoeff,
+                                                  Array<int> &attr)
+{
+   gfr_->ProjectBdrCoefficientTangent(real_vcoeff, attr);
+   gfi_->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
+}
+
+
+ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *f,
+                                     ComplexOperator::Convention convention)
+   : Vector(2*(f->GetVSize())),
+     conv_(convention)
+{
+   lfr_ = new LinearForm(f, &data[0]);
+   lfi_ = new LinearForm(f, &data[f->GetVSize()]);
+}
+
+ComplexLinearForm::~ComplexLinearForm()
+{
+   delete lfr_;
+   delete lfi_;
+}
+
+void
+ComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                                       LinearFormIntegrator *lfi_imag)
+{
+   if ( lfi_real ) { lfr_->AddDomainIntegrator(lfi_real); }
+   if ( lfi_imag ) { lfi_->AddDomainIntegrator(lfi_imag); }
+}
+
+void
+ComplexLinearForm::Update()
+{
+   FiniteElementSpace *fes = lfr_->FESpace();
+
+   this->Update(fes);
+}
+
+void
+ComplexLinearForm::Update(FiniteElementSpace *fes)
+{
+   int vsize = fes->GetVSize();
+   SetSize(2 * vsize);
+
+   Vector lfr(&data[0], vsize);
+   Vector lfi(&data[vsize], vsize);
+
+   lfr_->Update(fes, lfr, 0);
+   lfi_->Update(fes, lfi, 0);
+}
+
+void
+ComplexLinearForm::Assemble()
+{
+   lfr_->Assemble();
+   lfi_->Assemble();
+   if (conv_ == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      *lfi_ *= -1.0;
+   }
+}
+
+complex<double>
+ComplexLinearForm::operator()(const ComplexGridFunction &gf) const
+{
+   double s = (conv_ == ComplexOperator::HERMITIAN)?1.0:-1.0;
+   return complex<double>((*lfr_)(gf.real()) - s * (*lfi_)(gf.imag()),
+                          (*lfr_)(gf.imag()) + s * (*lfi_)(gf.real()));
+}
+
+
+SesquilinearForm::SesquilinearForm(FiniteElementSpace *f,
+                                   ComplexOperator::Convention convention)
+   : conv_(convention),
+     blfr_(new BilinearForm(f)),
+     blfi_(new BilinearForm(f))
+{}
+
+SesquilinearForm::~SesquilinearForm()
+{
+   delete blfr_;
+   delete blfi_;
+}
+
+void SesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                                           BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real) { blfr_->AddDomainIntegrator(bfi_real); }
+   if (bfi_imag) { blfi_->AddDomainIntegrator(bfi_imag); }
+}
+
+void
+SesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                                        BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real) { blfr_->AddBoundaryIntegrator(bfi_real); }
+   if (bfi_imag) { blfi_->AddBoundaryIntegrator(bfi_imag); }
+}
+
+void
+SesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                                        BilinearFormIntegrator *bfi_imag,
+                                        Array<int> & bdr_marker)
+{
+   if (bfi_real) { blfr_->AddBoundaryIntegrator(bfi_real, bdr_marker); }
+   if (bfi_imag) { blfi_->AddBoundaryIntegrator(bfi_imag, bdr_marker); }
+}
+
+void
+SesquilinearForm::Assemble(int skip_zeros)
+{
+   blfr_->Assemble(skip_zeros);
+   blfi_->Assemble(skip_zeros);
+}
+
+void
+SesquilinearForm::Finalize(int skip_zeros)
+{
+   blfr_->Finalize(skip_zeros);
+   blfi_->Finalize(skip_zeros);
+}
+
+ComplexSparseMatrix *
+SesquilinearForm::AssembleCompSpMat()
+{
+   return new ComplexSparseMatrix(&blfr_->SpMat(),
+                                  &blfi_->SpMat(),
+                                  false, false, conv_);
+
+}
+
+void
+SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                   Vector &x, Vector &b,
+                                   OperatorHandle &A,
+                                   Vector &X, Vector &B,
+                                   int ci)
+{
+   FiniteElementSpace * fes = blfr_->FESpace();
+
+   int vsize  = fes->GetVSize();
+   // int tvsize = pfes->GetTrueVSize();
+
+   double s = (conv_ == ComplexOperator::HERMITIAN)?1.0:-1.0;
+
+   // Allocate temporary vectors
+   Vector b_0(vsize);  b_0 = 0.0;
+   // Vector B_0(tvsize); B_0 = 0.0;
+
+   // Extract the real and imaginary parts of the input vectors
+   MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
+
+   MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
+   Vector b_r(b.GetData(), vsize);
+   Vector b_i(&(b.GetData())[vsize], vsize);
+   b_i *= s;
+   /*
+   X.SetSize(2 * tvsize);
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+
+   B.SetSize(2 * tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   */
+   SparseMatrix * A_r = new SparseMatrix;
+   SparseMatrix * A_i = new SparseMatrix;
+   Vector X_0, B_0;
+
+   b_0 = b_r;
+   blfr_->FormLinearSystem(ess_tdof_list, x_r, b_r, *A_r, X_0, B_0, ci);
+
+   int tvsize = B_0.Size();
+   X.SetSize(2 * tvsize);
+   B.SetSize(2 * tvsize);
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   X_r = X_0; B_r = B_0;
+
+   b_0 = 0.0;
+   blfi_->FormLinearSystem(ess_tdof_list, x_i, b_0, *A_i, X_0, B_0, false);
+   B_r -= B_0;
+
+   b_0 = b_i;
+   blfr_->FormLinearSystem(ess_tdof_list, x_i, b_0, *A_r, X_0, B_0, ci);
+   X_i = X_0; B_i = B_0;
+
+   b_0 = 0.0;
+   blfi_->FormLinearSystem(ess_tdof_list, x_r, b_0, *A_i, X_0, B_0, false);
+   B_i += B_0;
+
+   B_i *= s;
+   b_i *= s;
+
+   // A = A_r + i A_i
+   A.Clear();
+   ComplexSparseMatrix * A_sp =
+      new ComplexSparseMatrix(A_r, A_i, true, true, conv_);
+   A.Reset<ComplexSparseMatrix>(A_sp, true);
+}
+
+void
+SesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
+                                     Vector &x)
+{
+   FiniteElementSpace * fes = blfr_->FESpace();
+
+   const SparseMatrix *P = fes->GetConformingProlongation();
+
+   int vsize  = fes->GetVSize();
+   int tvsize = X.Size() / 2;
+
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
+
+   if (!P)
+   {
+      x = X;
+   }
+   else
+   {
+      // Apply conforming prolongation
+      P->Mult(X_r, x_r);
+      P->Mult(X_i, x_i);
+   }
+}
+
+void
+SesquilinearForm::Update(FiniteElementSpace *nfes)
+{
+   if ( blfr_ ) { blfr_->Update(nfes); }
+   if ( blfi_ ) { blfi_->Update(nfes); }
+}
+
+
+#ifdef MFEM_USE_MPI
+
+ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
+   : Vector(2*(pfes->GetVSize()))
+{
+   pgfr_ = new ParGridFunction(pfes, &data[0]);
+   pgfi_ = new ParGridFunction(pfes, &data[pfes->GetVSize()]);
+}
+
+void
+ParComplexGridFunction::Update()
+{
+   ParFiniteElementSpace * pfes = pgfr_->ParFESpace();
+
+   int vsize = pfes->GetVSize();
+
+   const Operator *T = pfes->GetUpdateOperator();
+   if (T)
+   {
+      // Update the individual GridFunction objects.  This will allocate
+      // new data arrays for each GridFunction.
+      pgfr_->Update();
+      pgfi_->Update();
+
+      // Our data array now contains old data as well as being the wrong size
+      // so reallocate it.
+      this->SetSize(2 * vsize);
+
+      // Create temporary vectors which point to the new data array
+      Vector gf_r(&data[0], vsize);
+      Vector gf_i(&data[vsize], vsize);
+
+      // Copy the updated GridFunctions into the new data array
+      gf_r = *pgfr_;
+      gf_i = *pgfi_;
+
+      // Replace the individual data arrays with pointers into the new data array
+      pgfr_->NewDataAndSize(&data[0], vsize);
+      pgfi_->NewDataAndSize(&data[vsize], vsize);
+   }
+   else
+   {
+      // The existing data will not be transferred to the new GridFunctions
+      // so delete it a allocate a new array
+      this->SetSize(2 * vsize);
+
+      // Point the individual GridFunctions to the new data array
+      pgfr_->NewDataAndSize(&data[0], vsize);
+      pgfi_->NewDataAndSize(&data[vsize], vsize);
+
+      // These updates will only set the proper 'sequence' value within
+      // the individual GridFunction objects because their sizes are
+      // already correct
+      pgfr_->Update();
+      pgfi_->Update();
+   }
+}
+
+void
+ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
+                                           Coefficient &imag_coeff)
+{
+   pgfr_->ProjectCoefficient(real_coeff);
+   pgfi_->ProjectCoefficient(imag_coeff);
+}
+
+void
+ParComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
+                                           VectorCoefficient &imag_vcoeff)
+{
+   pgfr_->ProjectCoefficient(real_vcoeff);
+   pgfi_->ProjectCoefficient(imag_vcoeff);
+}
+
+void
+ParComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
+                                              Coefficient &imag_coeff,
+                                              Array<int> &attr)
+{
+   pgfr_->ProjectBdrCoefficient(real_coeff, attr);
+   pgfi_->ProjectBdrCoefficient(imag_coeff, attr);
+}
+
+void
+ParComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient
+                                                    &real_vcoeff,
+                                                    VectorCoefficient
+                                                    &imag_vcoeff,
+                                                    Array<int> &attr)
+{
+   pgfr_->ProjectBdrCoefficientNormal(real_vcoeff, attr);
+   pgfi_->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
+}
+
+void
+ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
+                                                     &real_vcoeff,
+                                                     VectorCoefficient
+                                                     &imag_vcoeff,
+                                                     Array<int> &attr)
+{
+   pgfr_->ProjectBdrCoefficientTangent(real_vcoeff, attr);
+   pgfi_->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
+}
+
+void
+ParComplexGridFunction::Distribute(const Vector *tv)
+{
+   ParFiniteElementSpace * pfes = pgfr_->ParFESpace();
+   HYPRE_Int size = pfes->GetTrueVSize();
+
+   double * tvd = tv->GetData();
+   Vector tvr(tvd, size);
+   Vector tvi(&tvd[size], size);
+
+   pgfr_->Distribute(tvr);
+   pgfi_->Distribute(tvi);
+}
+
+void
+ParComplexGridFunction::ParallelProject(Vector &tv) const
+{
+   ParFiniteElementSpace * pfes = pgfr_->ParFESpace();
+   HYPRE_Int size = pfes->GetTrueVSize();
+
+   double * tvd = tv.GetData();
+   Vector tvr(tvd, size);
+   Vector tvi(&tvd[size], size);
+
+   pgfr_->ParallelProject(tvr);
+   pgfi_->ParallelProject(tvi);
+}
+
+
+ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
+                                           ComplexOperator::Convention
+                                           convention)
+   : Vector(2*(pfes->GetVSize())),
+     conv_(convention)
+{
+   plfr_ = new ParLinearForm(pfes, &data[0]);
+   plfi_ = new ParLinearForm(pfes, &data[pfes->GetVSize()]);
+
+   HYPRE_Int * tdof_offsets = pfes->GetTrueDofOffsets();
+
+   int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
+   tdof_offsets_ = new HYPRE_Int[n+1];
+
+   for (int i=0; i<=n; i++)
+   {
+      tdof_offsets_[i] = 2 * tdof_offsets[i];
+   }
+}
+
+ParComplexLinearForm::~ParComplexLinearForm()
+{
+   delete plfr_;
+   delete plfi_;
+   delete [] tdof_offsets_;
+}
+
+void
+ParComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                                          LinearFormIntegrator *lfi_imag)
+{
+   if ( lfi_real ) { plfr_->AddDomainIntegrator(lfi_real); }
+   if ( lfi_imag ) { plfi_->AddDomainIntegrator(lfi_imag); }
+}
+
+void
+ParComplexLinearForm::Update(ParFiniteElementSpace *pf)
+{
+   ParFiniteElementSpace *pfes = (pf!=NULL)?pf:plfr_->ParFESpace();
+   int vsize = pfes->GetVSize();
+   SetSize(2 * vsize);
+
+   Vector plfr(&data[0], vsize);
+   Vector plfi(&data[vsize], vsize);
+
+   plfr_->Update(pfes, plfr, 0);
+   plfi_->Update(pfes, plfi, 0);
+}
+
+void
+ParComplexLinearForm::Assemble()
+{
+   plfr_->Assemble();
+   plfi_->Assemble();
+   if (conv_ == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      *plfi_ *= -1.0;
+   }
+}
+
+void
+ParComplexLinearForm::ParallelAssemble(Vector &tv)
+{
+   HYPRE_Int size = plfr_->ParFESpace()->GetTrueVSize();
+
+   double * tvd = tv.GetData();
+   Vector tvr(tvd, size);
+   Vector tvi(&tvd[size], size);
+
+   plfr_->ParallelAssemble(tvr);
+   plfi_->ParallelAssemble(tvi);
+}
+
+HypreParVector *
+ParComplexLinearForm::ParallelAssemble()
+{
+   const ParFiniteElementSpace * pfes = plfr_->ParFESpace();
+
+   HypreParVector * tv = new HypreParVector(pfes->GetComm(),
+                                            2*(pfes->GlobalTrueVSize()),
+                                            tdof_offsets_);
+
+   HYPRE_Int size = pfes->GetTrueVSize();
+
+   double * tvd = tv->GetData();
+   Vector tvr(tvd, size);
+   Vector tvi(&tvd[size], size);
+
+   plfr_->ParallelAssemble(tvr);
+   plfi_->ParallelAssemble(tvi);
+
+   return tv;
+}
+
+complex<double>
+ParComplexLinearForm::operator()(const ParComplexGridFunction &gf) const
+{
+   double s = (conv_ == ComplexOperator::HERMITIAN)?1.0:-1.0;
+   return complex<double>((*plfr_)(gf.real()) - s * (*plfi_)(gf.imag()),
+                          (*plfr_)(gf.imag()) + s * (*plfi_)(gf.real()));
+}
+
+
+ParSesquilinearForm::ParSesquilinearForm(ParFiniteElementSpace *pf,
+                                         ComplexOperator::Convention
+                                         convention)
+   : conv_(convention),
+     pblfr_(new ParBilinearForm(pf)),
+     pblfi_(new ParBilinearForm(pf))
+{}
+
+ParSesquilinearForm::~ParSesquilinearForm()
+{
+   delete pblfr_;
+   delete pblfi_;
+}
+
+void ParSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                                              BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real) { pblfr_->AddDomainIntegrator(bfi_real); }
+   if (bfi_imag) { pblfi_->AddDomainIntegrator(bfi_imag); }
+}
+
+void
+ParSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                                           BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real) { pblfr_->AddBoundaryIntegrator(bfi_real); }
+   if (bfi_imag) { pblfi_->AddBoundaryIntegrator(bfi_imag); }
+}
+
+void
+ParSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                                           BilinearFormIntegrator *bfi_imag,
+                                           Array<int> & bdr_marker)
+{
+   if (bfi_real) { pblfr_->AddBoundaryIntegrator(bfi_real, bdr_marker); }
+   if (bfi_imag) { pblfi_->AddBoundaryIntegrator(bfi_imag, bdr_marker); }
+}
+
+void
+ParSesquilinearForm::Assemble(int skip_zeros)
+{
+   pblfr_->Assemble(skip_zeros);
+   pblfi_->Assemble(skip_zeros);
+}
+
+void
+ParSesquilinearForm::Finalize(int skip_zeros)
+{
+   pblfr_->Finalize(skip_zeros);
+   pblfi_->Finalize(skip_zeros);
+}
+
+ComplexHypreParMatrix *
+ParSesquilinearForm::ParallelAssemble()
+{
+   return new ComplexHypreParMatrix(pblfr_->ParallelAssemble(),
+                                    pblfi_->ParallelAssemble(),
+                                    true, true, conv_);
+
+}
+
+void
+ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                      Vector &x, Vector &b,
+                                      OperatorHandle &A,
+                                      Vector &X, Vector &B,
+                                      int ci)
+{
+   ParFiniteElementSpace * pfes = pblfr_->ParFESpace();
+
+   int tvs = pfes->TrueVSize();
+   cout << "TrueVSize returns " << tvs << endl;
+   cout << "GetVSize returns " << pfes->GetVSize() << endl;
+
+   int vsize = x.Size() / 2;
+   // int vsize  = pfes->GetVSize();
+   // int tvsize = pfes->GetTrueVSize();
+
+   cout << "x.Size/2 returns " << vsize << endl;
+
+   double s = (conv_ == ComplexOperator::HERMITIAN)?1.0:-1.0;
+
+   // Allocate temporary vectors
+   Vector b_0(vsize);  b_0 = 0.0;
+   // Vector B_0(tvsize); B_0 = 0.0;
+
+   // Extract the real and imaginary parts of the input vectors
+   // MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
+
+   MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
+   Vector b_r(b.GetData(), vsize);
+   Vector b_i(&(b.GetData())[vsize], vsize);
+   b_i *= s;
+   /*
+   X.SetSize(2 * tvsize);
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+
+   B.SetSize(2 * tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   */
+   OperatorHandle A_r, A_i;
+   Vector X_0, B_0;
+   cout << "pblfr fls 1" << endl << flush;
+   b_0 = b_r;
+   pblfr_->FormLinearSystem(ess_tdof_list, x_r, b_0, A_r, X_0, B_0, ci);
+
+   int tvsize = B_0.Size();
+   X.SetSize(2 * tvsize);
+   B.SetSize(2 * tvsize);
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   X_r = X_0; B_r = B_0;
+   cout << "pblfi fls 1" << endl << flush;
+   b_0 = 0.0;
+   pblfi_->FormLinearSystem(ess_tdof_list, x_i, b_0, A_i, X_0, B_0, false);
+   B_r -= B_0;
+   cout << "pblfr fls 2" << endl << flush;
+   b_0 = b_i;
+   pblfr_->FormLinearSystem(ess_tdof_list, x_i, b_0, A_r, X_0, B_0, ci);
+   X_i = X_0; B_i = B_0;
+   cout << "pblfi fls 2" << endl << flush;
+   b_0 = 0.0;
+   pblfi_->FormLinearSystem(ess_tdof_list, x_r, b_0, A_i, X_0, B_0, false);
+   B_i += B_0;
+
+   B_i *= s;
+   b_i *= s;
+
+   // A = A_r + i A_i
+   A.Clear();
+   if ( A_r.Type() == Operator::Hypre_ParCSR &&
+        A_i.Type() == Operator::Hypre_ParCSR )
+   {
+      ComplexHypreParMatrix * A_hyp =
+         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
+                                   A_i.As<HypreParMatrix>(),
+                                   A_r.OwnsOperator(),
+                                   A_i.OwnsOperator(),
+                                   conv_);
+      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.As<Operator>(),
+                             A_i.As<Operator>(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv_);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+}
+
+void
+ParSesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
+                                        Vector &x)
+{
+   ParFiniteElementSpace * pfes = pblfr_->ParFESpace();
+
+   const Operator &P = *pfes->GetProlongationMatrix();
+
+   int vsize  = pfes->GetVSize();
+   int tvsize = X.Size() / 2;
+
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
+
+   // Apply conforming prolongation
+   P.Mult(X_r, x_r);
+   P.Mult(X_i, x_i);
+}
+
+void
+ParSesquilinearForm::Update(FiniteElementSpace *nfes)
+{
+   if ( pblfr_ ) { pblfr_->Update(nfes); }
+   if ( pblfi_ ) { pblfi_->Update(nfes); }
+}
+
+
+#endif // MFEM_USE_MPI
+
+}

fem/complex_fem.hpp

@@ -0,0 +1,356 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_COMPLEX_FEM
+#define MFEM_COMPLEX_FEM
+
+#include "../linalg/complex_operator.hpp"
+#include "gridfunc.hpp"
+#include "linearform.hpp"
+#include "bilinearform.hpp"
+#ifdef MFEM_USE_MPI
+#include "pgridfunc.hpp"
+#include "plinearform.hpp"
+#include "pbilinearform.hpp"
+#endif
+#include <complex>
+
+namespace mfem
+{
+
+/// Class for complex-valued grid function - Vector with associated FE space.
+class ComplexGridFunction : public Vector
+{
+private:
+
+   GridFunction * gfr_;
+   GridFunction * gfi_;
+
+protected:
+   void Destroy() { delete gfr_; delete gfi_; }
+
+public:
+
+   /* @brief Construct a ComplexGridFunction associated with the
+      FiniteElementSpace @a *f. */
+   ComplexGridFunction(FiniteElementSpace *f);
+
+   void Update();
+
+   /// Assign constant values to the ComplexGridFunction data.
+   ComplexGridFunction &operator=(const std::complex<double> & value)
+   { *gfr_ = value.real(); *gfi_ = value.imag(); return *this; }
+
+   virtual void ProjectCoefficient(Coefficient &real_coeff,
+                                   Coefficient &imag_coeff);
+   virtual void ProjectCoefficient(VectorCoefficient &real_vcoeff,
+                                   VectorCoefficient &imag_vcoeff);
+
+   virtual void ProjectBdrCoefficient(Coefficient &real_coeff,
+                                      Coefficient &imag_coeff,
+                                      Array<int> &attr);
+   virtual void ProjectBdrCoefficientNormal(VectorCoefficient &real_coeff,
+                                            VectorCoefficient &imag_coeff,
+                                            Array<int> &attr);
+   virtual void ProjectBdrCoefficientTangent(VectorCoefficient &real_coeff,
+                                             VectorCoefficient &imag_coeff,
+                                             Array<int> &attr);
+
+   FiniteElementSpace *FESpace() { return gfr_->FESpace(); }
+   const FiniteElementSpace *FESpace() const { return gfr_->FESpace(); }
+
+   GridFunction & real() { return *gfr_; }
+   GridFunction & imag() { return *gfi_; }
+   const GridFunction & real() const { return *gfr_; }
+   const GridFunction & imag() const { return *gfi_; }
+
+   /// Destroys grid function.
+   virtual ~ComplexGridFunction() { Destroy(); }
+
+};
+
+class ComplexLinearForm : public Vector
+{
+private:
+   ComplexOperator::Convention conv_;
+
+protected:
+   LinearForm * lfr_;
+   LinearForm * lfi_;
+
+   // HYPRE_Int * tdof_offsets_;
+
+public:
+
+   ComplexLinearForm(FiniteElementSpace *fes,
+                     ComplexOperator::Convention
+                     convention = ComplexOperator::HERMITIAN);
+
+   virtual ~ComplexLinearForm();
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                            LinearFormIntegrator *lfi_imag);
+
+   FiniteElementSpace *FESpace() const { return lfr_->FESpace(); }
+
+   LinearForm & real() { return *lfr_; }
+   LinearForm & imag() { return *lfi_; }
+   const LinearForm & real() const { return *lfr_; }
+   const LinearForm & imag() const { return *lfi_; }
+
+   void Update();
+   void Update(FiniteElementSpace *f);
+
+   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
+   void Assemble();
+
+   std::complex<double> operator()(const ComplexGridFunction &gf) const;
+
+};
+
+// Class for sesquilinear form
+class SesquilinearForm
+{
+private:
+   ComplexOperator::Convention conv_;
+
+   //protected:
+   BilinearForm *blfr_;
+   BilinearForm *blfi_;
+
+public:
+   SesquilinearForm(FiniteElementSpace *fes,
+                    ComplexOperator::Convention
+                    convention = ComplexOperator::HERMITIAN);
+
+   ComplexOperator::Convention GetConvention() const { return conv_; }
+   void SetConvention(const ComplexOperator::Convention &
+                      convention) { conv_  = convention; }
+
+   BilinearForm & real() { return *blfr_; }
+   BilinearForm & imag() { return *blfi_; }
+   const BilinearForm & real() const { return *blfr_; }
+   const BilinearForm & imag() const { return *blfi_; }
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                            BilinearFormIntegrator *bfi_imag);
+
+   /// Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                              BilinearFormIntegrator *bfi_imag);
+
+   /// Adds new Boundary Integrator, restricted to specific boundary attributes.
+   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                              BilinearFormIntegrator *bfi_imag,
+                              Array<int> &bdr_marker);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   /// Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+   ComplexSparseMatrix *AssembleCompSpMat();
+
+   /// Return the parallel FE space associated with the ParBilinearForm.
+   FiniteElementSpace *FESpace() const { return blfr_->FESpace(); }
+
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0);
+
+   /** Call this method after solving a linear system constructed using the
+       FormLinearSystem method to recover the solution as a ParGridFunction-size
+       vector in x. Use the same arguments as in the FormLinearSystem call. */
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
+
+   virtual void Update(FiniteElementSpace *nfes = NULL);
+
+   virtual ~SesquilinearForm();
+};
+
+#ifdef MFEM_USE_MPI
+
+/// Class for complex-valued grid function - Vector with associated FE space.
+class ParComplexGridFunction : public Vector
+{
+private:
+
+   ParGridFunction * pgfr_;
+   ParGridFunction * pgfi_;
+
+protected:
+   void Destroy() { delete pgfr_; delete pgfi_; }
+
+public:
+
+   /* @brief Construct a ParComplexGridFunction associated with the
+      ParFiniteElementSpace @a *f. */
+   ParComplexGridFunction(ParFiniteElementSpace *pf);
+
+   void Update();
+
+   /// Assign constant values to the ParComplexGridFunction data.
+   ParComplexGridFunction &operator=(const std::complex<double> & value)
+   { *pgfr_ = value.real(); *pgfi_ = value.imag(); return *this; }
+
+   virtual void ProjectCoefficient(Coefficient &real_coeff,
+                                   Coefficient &imag_coeff);
+   virtual void ProjectCoefficient(VectorCoefficient &real_vcoeff,
+                                   VectorCoefficient &imag_vcoeff);
+
+   virtual void ProjectBdrCoefficient(Coefficient &real_coeff,
+                                      Coefficient &imag_coeff,
+                                      Array<int> &attr);
+   virtual void ProjectBdrCoefficientNormal(VectorCoefficient &real_coeff,
+                                            VectorCoefficient &imag_coeff,
+                                            Array<int> &attr);
+   virtual void ProjectBdrCoefficientTangent(VectorCoefficient &real_coeff,
+                                             VectorCoefficient &imag_coeff,
+                                             Array<int> &attr);
+
+   void Distribute(const Vector *tv);
+   void Distribute(const Vector &tv) { Distribute(&tv); }
+
+   /// Returns the vector restricted to the true dofs.
+   void ParallelProject(Vector &tv) const;
+
+   FiniteElementSpace *FESpace() { return pgfr_->FESpace(); }
+   const FiniteElementSpace *FESpace() const { return pgfr_->FESpace(); }
+
+   ParGridFunction & real() { return *pgfr_; }
+   ParGridFunction & imag() { return *pgfi_; }
+   const ParGridFunction & real() const { return *pgfr_; }
+   const ParGridFunction & imag() const { return *pgfi_; }
+
+   /// Destroys grid function.
+   virtual ~ParComplexGridFunction() { Destroy(); }
+
+};
+
+class ParComplexLinearForm : public Vector
+{
+private:
+   ComplexOperator::Convention conv_;
+
+protected:
+   ParLinearForm * plfr_;
+   ParLinearForm * plfi_;
+
+   HYPRE_Int * tdof_offsets_;
+
+public:
+
+   ParComplexLinearForm(ParFiniteElementSpace *pf,
+                        ComplexOperator::Convention
+                        convention = ComplexOperator::HERMITIAN);
+
+   virtual ~ParComplexLinearForm();
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                            LinearFormIntegrator *lfi_imag);
+
+   ParFiniteElementSpace *ParFESpace() const { return plfr_->ParFESpace(); }
+
+   ParLinearForm & real() { return *plfr_; }
+   ParLinearForm & imag() { return *plfi_; }
+   const ParLinearForm & real() const { return *plfr_; }
+   const ParLinearForm & imag() const { return *plfi_; }
+
+   void Update(ParFiniteElementSpace *pf = NULL);
+
+   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
+   void Assemble();
+
+   /// Assemble the vector on the true dofs, i.e. P^t v.
+   void ParallelAssemble(Vector &tv);
+
+   /// Returns the vector assembled on the true dofs, i.e. P^t v.
+   HypreParVector *ParallelAssemble();
+
+   std::complex<double> operator()(const ParComplexGridFunction &gf) const;
+
+};
+
+// Class for parallel sesquilinear form
+class ParSesquilinearForm
+{
+private:
+   ComplexOperator::Convention conv_;
+
+   //protected:
+   ParBilinearForm *pblfr_;
+   ParBilinearForm *pblfi_;
+
+public:
+   ParSesquilinearForm(ParFiniteElementSpace *pf,
+                       ComplexOperator::Convention
+                       convention = ComplexOperator::HERMITIAN);
+
+   ComplexOperator::Convention GetConvention() const { return conv_; }
+   void SetConvention(const ComplexOperator::Convention &
+                      convention) { conv_  = convention; }
+
+   ParBilinearForm & real() { return *pblfr_; }
+   ParBilinearForm & imag() { return *pblfi_; }
+   const ParBilinearForm & real() const { return *pblfr_; }
+   const ParBilinearForm & imag() const { return *pblfi_; }
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                            BilinearFormIntegrator *bfi_imag);
+
+   /// Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                              BilinearFormIntegrator *bfi_imag);
+
+   /// Adds new Boundary Integrator, restricted to specific boundary attributes.
+   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                              BilinearFormIntegrator *bfi_imag,
+                              Array<int> &bdr_marker);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   /// Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+   ComplexHypreParMatrix *ParallelAssemble();
+
+   /// Return the parallel FE space associated with the ParBilinearForm.
+   ParFiniteElementSpace *ParFESpace() const { return pblfr_->ParFESpace(); }
+
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0);
+
+   /** Call this method after solving a linear system constructed using the
+       FormLinearSystem method to recover the solution as a ParGridFunction-size
+       vector in x. Use the same arguments as in the FormLinearSystem call. */
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
+
+   virtual void Update(FiniteElementSpace *nfes = NULL);
+
+   virtual ~ParSesquilinearForm();
+};
+
+#endif // MFEM_USE_MPI
+
+}
+
+#endif // MFEM_COMPLEX_FEM

fem/datacollection.cpp

@@ -108,7 +108,6 @@ DataCollection::DataCollection(const std::string& collection_name, Mesh *mesh_)
    precision = precision_default;
    pad_digits_cycle = pad_digits_rank = pad_digits_default;
    format = SERIAL_FORMAT; // use serial mesh format
-   compression = false;
    error = NO_ERROR;
 }
 
@@ -162,14 +161,6 @@ void DataCollection::SetFormat(int fmt)
    format = fmt;
 }
 
-void DataCollection::SetCompression(bool comp)
-{
-   compression = comp;
-#ifdef MFEM_USE_GZSTREAM
-   MFEM_ASSERT(!compression, "GZStream not enabled in MFEM build.");
-#endif
-}
-
 void DataCollection::SetPrefixPath(const std::string& prefix)
 {
    if (!prefix.empty())
@@ -228,8 +219,7 @@ void DataCollection::SaveMesh()
    }
 
    std::string mesh_name = GetMeshFileName();
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream mesh_file(mesh_name.c_str(), mode);
+   std::ofstream mesh_file(mesh_name.c_str());
    mesh_file.precision(precision);
 #ifdef MFEM_USE_MPI
    const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
@@ -277,9 +267,7 @@ const
 
 void DataCollection::SaveOneField(const FieldMapIterator &it)
 {
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream field_file(GetFieldFileName(it->first).c_str(), mode);
-
+   std::ofstream field_file(GetFieldFileName(it->first).c_str());
    field_file.precision(precision);
    (it->second)->Save(field_file);
    if (!field_file)
@@ -291,8 +279,7 @@ void DataCollection::SaveOneField(const FieldMapIterator &it)
 
 void DataCollection::SaveOneQField(const QFieldMapIterator &it)
 {
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream q_field_file(GetFieldFileName(it->first).c_str(), mode);
+   std::ofstream q_field_file(GetFieldFileName(it->first).c_str());
    q_field_file.precision(precision);
    (it->second)->Save(q_field_file);
    if (!q_field_file)
@@ -589,7 +576,7 @@ void VisItDataCollection::LoadFields()
         it != field_info_map.end(); ++it)
    {
       std::string fname = path_left + it->first + path_right;
-      ifgzstream file(fname.c_str());
+      std::ifstream file(fname.c_str());
       // TODO: in parallel, check for errors on all processors
       if (!file)
       {

fem/datacollection.hpp

@@ -205,7 +205,6 @@ protected:
 
    /// Output mesh format: see the #Format enumeration
    int format;
-   bool compression;
 
    /// Should the collection delete its mesh and fields
    bool own_data;
@@ -347,9 +346,6 @@ public:
        validation. */
    virtual void SetFormat(int fmt);
 
-   /// Set the flag for use of gz compressed files
-   void SetCompression(bool comp);
-
    /// Set the path where the DataCollection will be saved.
    void SetPrefixPath(const std::string &prefix);
 

fem/fe.cpp

@@ -203,22 +203,6 @@ void FiniteElement::CalcPhysDShape(ElementTransformation &Trans,
    Mult(vshape, Trans.InverseJacobian(), dshape);
 }
 
-const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &,
-                                             DofToQuad::Mode) const
-{
-   mfem_error("FiniteElement::GetDofToQuad(...) is not implemented for "
-              "this element!");
-   return *dof2quad_array[0]; // suppress a warning
-}
-
-FiniteElement::~FiniteElement()
-{
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      delete dof2quad_array[i];
-   }
-}
-
 
 void ScalarFiniteElement::NodalLocalInterpolation (
    ElementTransformation &Trans, DenseMatrix &I,
@@ -294,95 +278,6 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
    }
 }
 
-const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
-                                                   DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = Dof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*Dof);
-   d2q->Bt.SetSize(Dof*nqpt);
-   d2q->G.SetSize(nqpt*Dim*Dof);
-   d2q->Gt.SetSize(Dof*nqpt*Dim);
-#ifdef MFEM_THREAD_SAFE
-   Vector c_shape(Dof);
-   DenseMatrix vshape(Dof, Dim);
-#endif
-   for (int i = 0; i < nqpt; i++)
-   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
-      CalcShape(ip, c_shape);
-      for (int j = 0; j < Dof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+Dof*i] = c_shape(j);
-      }
-      CalcDShape(ip, vshape);
-      for (int d = 0; d < Dim; d++)
-      {
-         for (int j = 0; j < Dof; j++)
-         {
-            d2q->G[i+nqpt*(d+Dim*j)] = d2q->Gt[j+Dof*(i+nqpt*d)] = vshape(j,d);
-         }
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-
-// protected method
-const DofToQuad &ScalarFiniteElement::GetTensorDofToQuad(
-   const TensorBasisElement &tb,
-   const IntegrationRule &ir, DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const Poly_1D::Basis &basis_1d = tb.GetBasis1D();
-   const int ndof = Order + 1;
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/Dim) + 0.5);
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis_1d.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-
 
 void NodalFiniteElement::ProjectCurl_2D(
    const FiniteElement &fe, ElementTransformation &Trans,

fem/fe.hpp

@@ -116,92 +116,7 @@ public:
    }
 };
 
-
-/** @brief Structure representing the matrices/tensors needed to evaluate (in
-    reference space) the values, gradients, divergences, or curls of a
-    FiniteElement at a the quadrature points of a given IntegrationRule. */
-/** Object of this type are typically created and owned by the respective
-    FiniteElement object. */
-class DofToQuad
-{
-public:
-   /// The FiniteElement that created and owns this object.
-   /** This pointer is not owned. */
-   const class FiniteElement *FE;
-
-   /** @brief IntegrationRule that defines the quadrature points at which the
-       basis functions of the #FE are evaluated. */
-   /** This pointer is not owned. */
-   const IntegrationRule *IntRule;
-
-   /// Type of data stored in the arrays #B, #Bt, #G, and #Gt.
-   enum Mode
-   {
-      /** @brief Full multidimensional representation which does not use tensor
-          product structure. The ordering of the degrees of freedom is as
-          defined by #FE */
-      FULL,
-
-      /** @brief Tensor product representation using 1D matrices/tensors with
-          dimensions using 1D number of quadrature points and degrees of
-          freedom. */
-      /** When representing a vector-valued FiniteElement, two DofToQuad objects
-          are used to describe the "closed" and "open" 1D basis functions
-          (TODO). */
-      TENSOR
-   };
-
-   /// Describes the contents of the #B, #Bt, #G, and #Gt arrays, see #Mode.
-   Mode mode;
-
-   /** @brief Number of degrees of freedom = number of basis functions. When
-       #mode is TENSOR, this is the 1D number. */
-   int ndof;
-
-   /** @brief Number of quadrature points. When #mode is TENSOR, this is the 1D
-       number. */
-   int nqpt;
-
-   /// Basis functions evaluated at quadrature points.
-   /** The storage layout is column-major with dimensions:
-       - #nqpt x #ndof, for scalar elements, or
-       - #nqpt x dim x #ndof, for vector elements, (TODO)
-
-       where
-
-       - dim = dimension of the finite element reference space when #mode is
-         FULL, and dim = 1 when #mode is TENSOR. */
-   Array<double> B;
-
-   /// Transpose of #B.
-   /** The storage layout is column-major with dimensions:
-       - #ndof x #nqpt, for scalar elements, or
-       - #ndof x #nqpt x dim, for vector elements (TODO). */
-   Array<double> Bt;
-
-   /** @brief Gradients/divergences/curls of basis functions evaluated at
-       quadrature points. */
-   /** The storage layout is column-major with dimensions:
-       - #nqpt x dim x #ndof, for scalar elements, or
-       - #nqpt x #ndof, for H(div) vector elements (TODO), or
-       - #nqpt x cdim x #ndof, for H(curl) vector elements (TODO),
-
-       where
-
-       - dim = dimension of the finite element reference space when #mode is
-         FULL, and 1 when #mode is TENSOR,
-       - cdim = 1/1/3 in 1D/2D/3D, respectively, when #mode is FULL, and cdim =
-         1 when #mode is TENSOR. */
-   Array<double> G;
-
-   /// Transpose of #G.
-   /** The storage layout is column-major with dimensions:
-       - #ndof x #nqpt x dim, for scalar elements, or
-       - #ndof x #nqpt, for H(div) vector elements (TODO), or
-       - #ndof x #nqpt x cdim, for H(curl) vector elements (TODO). */
-   Array<double> Gt;
-};
-
+// Base and derived classes for finite elements
 
 /// Describes the space on each element
 class FunctionSpace
@@ -221,10 +136,6 @@ class VectorCoefficient;
 class MatrixCoefficient;
 class KnotVector;
 
-
-// Base and derived classes for finite elements
-
-
 /// Abstract class for Finite Elements
 class FiniteElement
 {
@@ -241,10 +152,6 @@ protected:
 #ifndef MFEM_THREAD_SAFE
    mutable DenseMatrix vshape; // Dof x Dim
 #endif
-   /// Container for all DofToQuad objects created by the FiniteElement.
-   /** Multiple DofToQuad objects may be needed when different quadrature rules
-       or different DofToQuad::Mode are used. */
-   mutable Array<DofToQuad*> dof2quad_array;
 
 public:
    /// Enumeration for RangeType and DerivRangeType
@@ -510,13 +417,7 @@ public:
                            ElementTransformation &Trans,
                            DenseMatrix &div) const;
 
-   /** Return a DofToQuad structure corresponding to the given IntegrationRule
-       using the given DofToQuad::Mode. */
-   /** See the documentation for DofToQuad for more details. */
-   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                         DofToQuad::Mode mode) const;
-
-   virtual ~FiniteElement();
+   virtual ~FiniteElement () { }
 
    static bool IsClosedType(int b_type)
    {
@@ -563,10 +464,6 @@ protected:
       return static_cast<const ScalarFiniteElement &>(fe);
    }
 
-   const DofToQuad &GetTensorDofToQuad(const class TensorBasisElement &tb,
-                                       const IntegrationRule &ir,
-                                       DofToQuad::Mode mode) const;
-
 public:
    ScalarFiniteElement(int D, Geometry::Type G, int Do, int O,
                        int F = FunctionSpace::Pk)
@@ -597,9 +494,6 @@ public:
    void ScalarLocalInterpolation(ElementTransformation &Trans,
                                  DenseMatrix &I,
                                  const ScalarFiniteElement &fine_fe) const;
-
-   virtual const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                         DofToQuad::Mode mode) const;
 };
 
 class NodalFiniteElement : public ScalarFiniteElement
@@ -1856,14 +1750,6 @@ class NodalTensorFiniteElement : public NodalFiniteElement,
 public:
    NodalTensorFiniteElement(const int dims, const int p, const int btype,
                             const DofMapType dmtype);
-
-   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const
-   {
-      return (mode == DofToQuad::FULL) ?
-             ScalarFiniteElement::GetDofToQuad(ir, mode) :
-             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
-   }
 };
 
 class PositiveTensorFiniteElement : public PositiveFiniteElement,
@@ -1872,14 +1758,6 @@ class PositiveTensorFiniteElement : public PositiveFiniteElement,
 public:
    PositiveTensorFiniteElement(const int dims, const int p,
                                const DofMapType dmtype);
-
-   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
-                                 DofToQuad::Mode mode) const
-   {
-      return (mode == DofToQuad::FULL) ?
-             ScalarFiniteElement::GetDofToQuad(ir, mode) :
-             ScalarFiniteElement::GetTensorDofToQuad(*this, ir, mode);
-   }
 };
 
 class H1_SegmentElement : public NodalTensorFiniteElement

fem/fem.hpp

@@ -18,6 +18,7 @@
 #include "fe_coll.hpp"
 #include "eltrans.hpp"
 #include "coefficient.hpp"
+#include "complex_fem.hpp"
 #include "lininteg.hpp"
 #include "nonlininteg.hpp"
 #include "bilininteg.hpp"

fem/fespace.cpp

@@ -12,7 +12,6 @@
 // Implementation of FiniteElementSpace
 
 #include "../general/text.hpp"
-#include "../general/forall.hpp"
 #include "../mesh/mesh_headers.hpp"
 #include "fem.hpp"
 
@@ -386,7 +385,6 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
                                       Array<int> &list)
 {
    int num_marked = 0;
-   marker.HostRead(); // make sure we can read the array on host
    for (int i = 0; i < marker.Size(); i++)
    {
       if (marker[i]) { num_marked++; }
@@ -654,9 +652,9 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    // create the conforming restriction matrix cR
    int *cR_J;
    {
-      int *cR_I = new int[n_true_dofs+1];
-      double *cR_A = new double[n_true_dofs];
-      cR_J = new int[n_true_dofs];
+      int *cR_I = mfem::New<int>(n_true_dofs+1);
+      double *cR_A = mfem::New<double>(n_true_dofs);
+      cR_J = mfem::New<int>(n_true_dofs);
       for (int i = 0; i < n_true_dofs; i++)
       {
          cR_I[i] = i;
@@ -734,8 +732,6 @@ void FiniteElementSpace::BuildConformingInterpolation() const
       MakeVDimMatrix(*cP);
       MakeVDimMatrix(*cR);
    }
-
-   if (Device::IsEnabled()) { cP->BuildTranspose(); }
 }
 
 void FiniteElementSpace::MakeVDimMatrix(SparseMatrix &mat) const
@@ -786,57 +782,6 @@ int FiniteElementSpace::GetNConformingDofs() const
    return P ? (P->Width() / vdim) : ndofs;
 }
 
-const Operator *FiniteElementSpace::GetElementRestriction(
-   ElementDofOrdering e_ordering) const
-{
-   // Check if we have a discontinuous space using the FE collection:
-   const L2_FECollection *dg_space = dynamic_cast<const L2_FECollection*>(fec);
-   if (dg_space) { return NULL; }
-   // TODO: support other DG collections.
-   if (e_ordering == ElementDofOrdering::LEXICOGRAPHIC)
-   {
-      if (L2E_lex.Ptr() == NULL)
-      {
-         L2E_lex.Reset(new ElementRestriction(*this, e_ordering));
-      }
-      return L2E_lex.Ptr();
-   }
-   // e_ordering == ElementDofOrdering::NATIVE
-   if (L2E_nat.Ptr() == NULL)
-   {
-      L2E_nat.Reset(new ElementRestriction(*this, e_ordering));
-   }
-   return L2E_nat.Ptr();
-}
-
-const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
-   const IntegrationRule &ir) const
-{
-   for (int i = 0; i < E2Q_array.Size(); i++)
-   {
-      const QuadratureInterpolator *qi = E2Q_array[i];
-      if (qi->IntRule == &ir) { return qi; }
-   }
-
-   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, ir);
-   E2Q_array.Append(qi);
-   return qi;
-}
-
-const QuadratureInterpolator *FiniteElementSpace::GetQuadratureInterpolator(
-   const QuadratureSpace &qs) const
-{
-   for (int i = 0; i < E2Q_array.Size(); i++)
-   {
-      const QuadratureInterpolator *qi = E2Q_array[i];
-      if (qi->qspace == &qs) { return qi; }
-   }
-
-   QuadratureInterpolator *qi = new QuadratureInterpolator(*this, qs);
-   E2Q_array.Append(qi);
-   return qi;
-}
-
 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
    const int coarse_ndofs, const Table &coarse_elem_dof,
    const DenseTensor localP[]) const
@@ -1540,10 +1485,6 @@ void FiniteElementSpace::GetElementDofs (int i, Array<int> &dofs) const
 
 const FiniteElement *FiniteElementSpace::GetFE(int i) const
 {
-   if (i < 0 || !mesh->GetNE()) { return NULL; }
-   MFEM_VERIFY(i < mesh->GetNE(),
-               "Invalid element id " << i << ", maximum allowed " << mesh->GetNE()-1);
-
    const FiniteElement *FE =
       fec->FiniteElementForGeometry(mesh->GetElementBaseGeometry(i));
 
@@ -1848,13 +1789,6 @@ void FiniteElementSpace::Destroy()
    delete cR;
    delete cP;
    Th.Clear();
-   L2E_nat.Clear();
-   L2E_lex.Clear();
-   for (int i = 0; i < E2Q_array.Size(); i++)
-   {
-      delete E2Q_array[i];
-   }
-   E2Q_array.SetSize(0);
 
    dof_elem_array.DeleteAll();
    dof_ldof_array.DeleteAll();
@@ -2416,7 +2350,8 @@ const Operator &InterpolationGridTransfer::BackwardOperator()
       return *B.Ptr();
    }
 
-   // Construct B, if not set, define a suitable mass_integ
+   // Construct B
+   // If not set, define a suitable mass_integ
    if (!mass_integ && ran_fes.GetNE() > 0)
    {
       const FiniteElement *f_fe_0 = ran_fes.GetFE(0);
@@ -2579,7 +2514,7 @@ void L2ProjectionGridTransfer::L2Projection::Mult(
       fes_ho.GetElementVDofs(iho, vdofs);
       x.GetSubVector(vdofs, xel_mat.GetData());
       mfem::Mult(R(iho), xel_mat, yel_mat);
-      // Place result correctly into the low-order vector
+      // Place result correctly into low-order vector
       for (int iref=0; iref<nref; ++iref)
       {
          int ilor = ho2lor.GetRow(iho)[iref];
@@ -2637,490 +2572,4 @@ const Operator &L2ProjectionGridTransfer::BackwardOperator()
    return *B;
 }
 
-
-ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
-                                       ElementDofOrdering e_ordering)
-   : fes(f),
-     ne(fes.GetNE()),
-     vdim(fes.GetVDim()),
-     byvdim(fes.GetOrdering() == Ordering::byVDIM),
-     ndofs(fes.GetNDofs()),
-     dof(ne > 0 ? fes.GetFE(0)->GetDof() : 0),
-     nedofs(ne*dof),
-     offsets(ndofs+1),
-     indices(ne*dof)
-{
-   // Assuming all finite elements are the same.
-   height = vdim*ne*dof;
-   width = fes.GetVSize();
-   const bool dof_reorder = (e_ordering == ElementDofOrdering::LEXICOGRAPHIC);
-   const int *dof_map = NULL;
-   if (dof_reorder && ne > 0)
-   {
-      for (int e = 0; e < ne; ++e)
-      {
-         const FiniteElement *fe = fes.GetFE(e);
-         const TensorBasisElement* el =
-            dynamic_cast<const TensorBasisElement*>(fe);
-         if (el) { continue; }
-         mfem_error("Finite element not suitable for lexicographic ordering");
-      }
-      const FiniteElement *fe = fes.GetFE(0);
-      const TensorBasisElement* el =
-         dynamic_cast<const TensorBasisElement*>(fe);
-      const Array<int> &fe_dof_map = el->GetDofMap();
-      MFEM_VERIFY(fe_dof_map.Size() > 0, "invalid dof map");
-      dof_map = fe_dof_map.GetData();
-   }
-   const Table& e2dTable = fes.GetElementToDofTable();
-   const int* elementMap = e2dTable.GetJ();
-   // We will be keeping a count of how many local nodes point to its global dof
-   for (int i = 0; i <= ndofs; ++i)
-   {
-      offsets[i] = 0;
-   }
-   for (int e = 0; e < ne; ++e)
-   {
-      for (int d = 0; d < dof; ++d)
-      {
-         const int gid = elementMap[dof*e + d];
-         ++offsets[gid + 1];
-      }
-   }
-   // Aggregate to find offsets for each global dof
-   for (int i = 1; i <= ndofs; ++i)
-   {
-      offsets[i] += offsets[i - 1];
-   }
-   // For each global dof, fill in all local nodes that point to it
-   for (int e = 0; e < ne; ++e)
-   {
-      for (int d = 0; d < dof; ++d)
-      {
-         const int did = (!dof_reorder)?d:dof_map[d];
-         const int gid = elementMap[dof*e + did];
-         const int lid = dof*e + d;
-         indices[offsets[gid]++] = lid;
-      }
-   }
-   // We shifted the offsets vector by 1 by using it as a counter.
-   // Now we shift it back.
-   for (int i = ndofs; i > 0; --i)
-   {
-      offsets[i] = offsets[i - 1];
-   }
-   offsets[0] = 0;
-}
-
-void ElementRestriction::Mult(const Vector& x, Vector& y) const
-{
-   // Assumes all elements have the same number of dofs
-   const int nd = dof;
-   const int vd = vdim;
-   const bool t = byvdim;
-   auto d_offsets = offsets.Read();
-   auto d_indices = indices.Read();
-   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_y = Reshape(y.Write(), nd, vd, ne);
-   MFEM_FORALL(i, ndofs,
-   {
-      const int offset = d_offsets[i];
-      const int nextOffset = d_offsets[i+1];
-      for (int c = 0; c < vd; ++c)
-      {
-         const double dofValue = d_x(t?c:i,t?i:c);
-         for (int j = offset; j < nextOffset; ++j)
-         {
-            const int idx_j = d_indices[j];
-            d_y(idx_j % nd, c, idx_j / nd) = dofValue;
-         }
-      }
-   });
-}
-
-void ElementRestriction::MultTranspose(const Vector& x, Vector& y) const
-{
-   // Assumes all elements have the same number of dofs
-   const int nd = dof;
-   const int vd = vdim;
-   const bool t = byvdim;
-   auto d_offsets = offsets.Read();
-   auto d_indices = indices.Read();
-   auto d_x = Reshape(x.Read(), nd, vd, ne);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
-   MFEM_FORALL(i, ndofs,
-   {
-      const int offset = d_offsets[i];
-      const int nextOffset = d_offsets[i + 1];
-      for (int c = 0; c < vd; ++c)
-      {
-         double dofValue = 0;
-         for (int j = offset; j < nextOffset; ++j)
-         {
-            const int idx_j = d_indices[j];
-            dofValue +=  d_x(idx_j % nd, c, idx_j / nd);
-         }
-         d_y(t?c:i,t?i:c) = dofValue;
-      }
-   });
-}
-
-
-QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
-                                               const IntegrationRule &ir)
-{
-   fespace = &fes;
-   qspace = NULL;
-   IntRule = &ir;
-   use_tensor_products = true; // not implemented yet (not used)
-
-   if (fespace->GetNE() == 0) { return; }
-   const FiniteElement *fe = fespace->GetFE(0);
-   MFEM_VERIFY(dynamic_cast<const ScalarFiniteElement*>(fe) != NULL,
-               "Only scalar finite elements are supported");
-}
-
-QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
-                                               const QuadratureSpace &qs)
-{
-   fespace = &fes;
-   qspace = &qs;
-   IntRule = NULL;
-   use_tensor_products = true; // not implemented yet (not used)
-
-   if (fespace->GetNE() == 0) { return; }
-   const FiniteElement *fe = fespace->GetFE(0);
-   MFEM_VERIFY(dynamic_cast<const ScalarFiniteElement*>(fe) != NULL,
-               "Only scalar finite elements are supported");
-}
-
-template<const int T_VDIM, const int T_ND, const int T_NQ>
-void QuadratureInterpolator::Eval2D(
-   const int NE,
-   const int vdim,
-   const DofToQuad &maps,
-   const Vector &e_vec,
-   Vector &q_val,
-   Vector &q_der,
-   Vector &q_det,
-   const int eval_flags)
-{
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_VERIFY(ND <= MAX_ND2D, "");
-   MFEM_VERIFY(NQ <= MAX_NQ2D, "");
-   MFEM_VERIFY(VDIM == 2 || !(eval_flags & DETERMINANTS), "");
-   auto B = Reshape(maps.B.Read(), NQ, ND);
-   auto G = Reshape(maps.G.Read(), NQ, 2, ND);
-   auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
-   auto val = Reshape(q_val.Write(), NQ, VDIM, NE);
-   auto der = Reshape(q_der.Write(), NQ, VDIM, 2, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : MAX_ND2D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : MAX_VDIM2D;
-      double s_E[max_VDIM*max_ND];
-      for (int d = 0; d < ND; d++)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c+d*VDIM] = E(d,c,e);
-         }
-      }
-      for (int q = 0; q < NQ; ++q)
-      {
-         if (eval_flags & VALUES)
-         {
-            double ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const double b = B(q,d);
-               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
-            }
-            for (int c = 0; c < VDIM; c++) { val(q,c,e) = ed[c]; }
-         }
-         if ((eval_flags & DERIVATIVES) || (eval_flags & DETERMINANTS))
-         {
-            // use MAX_VDIM2D to avoid "subscript out of range" warnings
-            double D[MAX_VDIM2D*2];
-            for (int i = 0; i < 2*VDIM; i++) { D[i] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const double wx = G(q,0,d);
-               const double wy = G(q,1,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  double s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-               }
-            }
-            if (eval_flags & DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  der(q,c,0,e) = D[c+VDIM*0];
-                  der(q,c,1,e) = D[c+VDIM*1];
-               }
-            }
-            if (VDIM == 2 && (eval_flags & DETERMINANTS))
-            {
-               // The check (VDIM == 2) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 2).
-               det(q,e) = D[0]*D[3] - D[1]*D[2];
-            }
-         }
-      }
-   });
-}
-
-template<const int T_VDIM, const int T_ND, const int T_NQ>
-void QuadratureInterpolator::Eval3D(
-   const int NE,
-   const int vdim,
-   const DofToQuad &maps,
-   const Vector &e_vec,
-   Vector &q_val,
-   Vector &q_der,
-   Vector &q_det,
-   const int eval_flags)
-{
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_VERIFY(ND <= MAX_ND3D, "");
-   MFEM_VERIFY(NQ <= MAX_NQ3D, "");
-   MFEM_VERIFY(VDIM == 3 || !(eval_flags & DETERMINANTS), "");
-   auto B = Reshape(maps.B.Read(), NQ, ND);
-   auto G = Reshape(maps.G.Read(), NQ, 3, ND);
-   auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
-   auto val = Reshape(q_val.Write(), NQ, VDIM, NE);
-   auto der = Reshape(q_der.Write(), NQ, VDIM, 3, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : MAX_ND3D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : MAX_VDIM3D;
-      double s_E[max_VDIM*max_ND];
-      for (int d = 0; d < ND; d++)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c+d*VDIM] = E(d,c,e);
-         }
-      }
-      for (int q = 0; q < NQ; ++q)
-      {
-         if (eval_flags & VALUES)
-         {
-            double ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const double b = B(q,d);
-               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
-            }
-            for (int c = 0; c < VDIM; c++) { val(q,c,e) = ed[c]; }
-         }
-         if ((eval_flags & DERIVATIVES) || (eval_flags & DETERMINANTS))
-         {
-            // use MAX_VDIM3D to avoid "subscript out of range" warnings
-            double D[MAX_VDIM3D*3];
-            for (int i = 0; i < 3*VDIM; i++) { D[i] = 0.0; }
-            for (int d = 0; d < ND; ++d)
-            {
-               const double wx = G(q,0,d);
-               const double wy = G(q,1,d);
-               const double wz = G(q,2,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  double s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-                  D[c+VDIM*2] += s_e * wz;
-               }
-            }
-            if (eval_flags & DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  der(q,c,0,e) = D[c+VDIM*0];
-                  der(q,c,1,e) = D[c+VDIM*1];
-                  der(q,c,2,e) = D[c+VDIM*2];
-               }
-            }
-            if (VDIM == 3 && (eval_flags & DETERMINANTS))
-            {
-               // The check (VDIM == 3) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 3).
-               det(q,e) = D[0] * (D[4] * D[8] - D[5] * D[7]) +
-                          D[3] * (D[2] * D[7] - D[1] * D[8]) +
-                          D[6] * (D[1] * D[5] - D[2] * D[4]);
-            }
-         }
-      }
-   });
-}
-
-void QuadratureInterpolator::Mult(
-   const Vector &e_vec, unsigned eval_flags,
-   Vector &q_val, Vector &q_der, Vector &q_det) const
-{
-   const int ne = fespace->GetNE();
-   if (ne == 0) { return; }
-   const int vdim = fespace->GetVDim();
-   const int dim = fespace->GetMesh()->Dimension();
-   const FiniteElement *fe = fespace->GetFE(0);
-   const IntegrationRule *ir =
-      IntRule ? IntRule : &qspace->GetElementIntRule(0);
-   const DofToQuad &maps = fe->GetDofToQuad(*ir, DofToQuad::FULL);
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   void (*eval_func)(
-      const int NE,
-      const int vdim,
-      const DofToQuad &maps,
-      const Vector &e_vec,
-      Vector &q_val,
-      Vector &q_der,
-      Vector &q_det,
-      const int eval_flags) = NULL;
-   if (vdim == 1)
-   {
-      if (dim == 2)
-      {
-         switch (100*nd + nq)
-         {
-            // Q0
-            case 101: eval_func = &Eval2D<1,1,1>; break;
-            case 104: eval_func = &Eval2D<1,1,4>; break;
-            // Q1
-            case 404: eval_func = &Eval2D<1,4,4>; break;
-            case 409: eval_func = &Eval2D<1,4,9>; break;
-            // Q2
-            case 909: eval_func = &Eval2D<1,9,9>; break;
-            case 916: eval_func = &Eval2D<1,9,16>; break;
-            // Q3
-            case 1616: eval_func = &Eval2D<1,16,16>; break;
-            case 1625: eval_func = &Eval2D<1,16,25>; break;
-            case 1636: eval_func = &Eval2D<1,16,36>; break;
-            // Q4
-            case 2525: eval_func = &Eval2D<1,25,25>; break;
-            case 2536: eval_func = &Eval2D<1,25,36>; break;
-            case 2549: eval_func = &Eval2D<1,25,49>; break;
-            case 2564: eval_func = &Eval2D<1,25,64>; break;
-         }
-         if (nq >= 100 || !eval_func)
-         {
-            eval_func = &Eval2D<1>;
-         }
-      }
-      else if (dim == 3)
-      {
-         switch (1000*nd + nq)
-         {
-            // Q0
-            case 1001: eval_func = &Eval3D<1,1,1>; break;
-            case 1008: eval_func = &Eval3D<1,1,8>; break;
-            // Q1
-            case 8008: eval_func = &Eval3D<1,8,8>; break;
-            case 8027: eval_func = &Eval3D<1,8,27>; break;
-            // Q2
-            case 27027: eval_func = &Eval3D<1,27,27>; break;
-            case 27064: eval_func = &Eval3D<1,27,64>; break;
-            // Q3
-            case 64064: eval_func = &Eval3D<1,64,64>; break;
-            case 64125: eval_func = &Eval3D<1,64,125>; break;
-            case 64216: eval_func = &Eval3D<1,64,216>; break;
-            // Q4
-            case 125125: eval_func = &Eval3D<1,125,125>; break;
-            case 125216: eval_func = &Eval3D<1,125,216>; break;
-         }
-         if (nq >= 1000 || !eval_func)
-         {
-            eval_func = &Eval3D<1>;
-         }
-      }
-   }
-   else if (vdim == dim)
-   {
-      if (dim == 2)
-      {
-         switch (100*nd + nq)
-         {
-            // Q1
-            case 404: eval_func = &Eval2D<2,4,4>; break;
-            case 409: eval_func = &Eval2D<2,4,9>; break;
-            // Q2
-            case 909: eval_func = &Eval2D<2,9,9>; break;
-            case 916: eval_func = &Eval2D<2,9,16>; break;
-            // Q3
-            case 1616: eval_func = &Eval2D<2,16,16>; break;
-            case 1625: eval_func = &Eval2D<2,16,25>; break;
-            case 1636: eval_func = &Eval2D<2,16,36>; break;
-            // Q4
-            case 2525: eval_func = &Eval2D<2,25,25>; break;
-            case 2536: eval_func = &Eval2D<2,25,36>; break;
-            case 2549: eval_func = &Eval2D<2,25,49>; break;
-            case 2564: eval_func = &Eval2D<2,25,64>; break;
-         }
-         if (nq >= 100 || !eval_func)
-         {
-            eval_func = &Eval2D<2>;
-         }
-      }
-      else if (dim == 3)
-      {
-         switch (1000*nd + nq)
-         {
-            // Q1
-            case 8008: eval_func = &Eval3D<3,8,8>; break;
-            case 8027: eval_func = &Eval3D<3,8,27>; break;
-            // Q2
-            case 27027: eval_func = &Eval3D<3,27,27>; break;
-            case 27064: eval_func = &Eval3D<3,27,64>; break;
-            // Q3
-            case 64064: eval_func = &Eval3D<3,64,64>; break;
-            case 64125: eval_func = &Eval3D<3,64,125>; break;
-            case 64216: eval_func = &Eval3D<3,64,216>; break;
-            // Q4
-            case 125125: eval_func = &Eval3D<3,125,125>; break;
-            case 125216: eval_func = &Eval3D<3,125,216>; break;
-         }
-         if (nq >= 1000 || !eval_func)
-         {
-            eval_func = &Eval3D<3>;
-         }
-      }
-   }
-   if (eval_func)
-   {
-      eval_func(ne, vdim, maps, e_vec, q_val, q_der, q_det, eval_flags);
-   }
-   else
-   {
-      MFEM_ABORT("case not supported yet");
-   }
-}
-
-void QuadratureInterpolator::MultTranspose(
-   unsigned eval_flags, const Vector &q_val, const Vector &q_der,
-   Vector &e_vec) const
-{
-   MFEM_ABORT("this method is not implemented yet");
-}
-
 } // namespace mfem

fem/fespace.hpp

@@ -59,25 +59,9 @@ Ordering::Map<Ordering::byVDIM>(int ndofs, int vdim, int dof, int vd)
 }
 
 
-/// Constants describing the possible orderings of the DOFs in one element.
-enum class ElementDofOrdering
-{
-   /// Native ordering as defined by the FiniteElement.
-   /** This ordering can be used by tensor-product elements when the
-       interpolation from the DOFs to quadrature points does not use the
-       tensor-product structure. */
-   NATIVE,
-   /// Lexicographic ordering for tensor-product FiniteElements.
-   /** This ordering can be used only with tensor-product elements. */
-   LEXICOGRAPHIC
-};
-
-
 // Forward declarations
 class NURBSExtension;
 class BilinearFormIntegrator;
-class QuadratureSpace;
-class QuadratureInterpolator;
 
 
 /** @brief Class FiniteElementSpace - responsible for providing FEM view of the
@@ -126,11 +110,6 @@ protected:
    /// Transformation to apply to GridFunctions after space Update().
    OperatorHandle Th;
 
-   /// The element restriction operators, see GetElementRestriction().
-   mutable OperatorHandle L2E_nat, L2E_lex;
-
-   mutable Array<QuadratureInterpolator*> E2Q_array;
-
    long sequence; // should match Mesh::GetSequence
 
    void UpdateNURBS();
@@ -278,60 +257,14 @@ public:
    bool Conforming() const { return mesh->Conforming(); }
    bool Nonconforming() const { return mesh->Nonconforming(); }
 
-   /// The returned SparseMatrix is owned by the FiniteElementSpace.
    const SparseMatrix *GetConformingProlongation() const;
-
-   /// The returned SparseMatrix is owned by the FiniteElementSpace.
    const SparseMatrix *GetConformingRestriction() const;
 
-   /// The returned Operator is owned by the FiniteElementSpace.
    virtual const Operator *GetProlongationMatrix() const
    { return GetConformingProlongation(); }
-
-   /// The returned SparseMatrix is owned by the FiniteElementSpace.
    virtual const SparseMatrix *GetRestrictionMatrix() const
    { return GetConformingRestriction(); }
 
-   /// Return an Operator that converts L-vectors to E-vectors.
-   /** An L-vector is a vector of size GetVSize() which is the same size as a
-       GridFunction. An E-vector represents the element-wise discontinuous
-       version of the FE space.
-
-       The layout of the E-vector is: ND x VDIM x NE, where ND is the number of
-       degrees of freedom, VDIM is the vector dimension of the FE space, and NE
-       is the number of the mesh elements.
-
-       The parameter @a e_ordering describes how the local DOFs in each element
-       should be ordered, see ElementDofOrdering.
-
-       For discontinuous spaces, where the element-restriction is the identity,
-       this method will return NULL.
-
-       The returned Operator is owned by the FiniteElementSpace. */
-   const Operator *GetElementRestriction(ElementDofOrdering e_ordering) const;
-
-   /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
-       quadrature point values and/or derivatives (Q-vectors). */
-   /** An E-vector represents the element-wise discontinuous version of the FE
-       space and can be obtained, for example, from a GridFunction using the
-       Operator returned by GetElementRestriction().
-
-       All elements will use the same IntegrationRule, @a ir as the target
-       quadrature points. */
-   const QuadratureInterpolator *GetQuadratureInterpolator(
-      const IntegrationRule &ir) const;
-
-   /** @brief Return a QuadratureInterpolator that interpolates E-vectors to
-       quadrature point values and/or derivatives (Q-vectors). */
-   /** An E-vector represents the element-wise discontinuous version of the FE
-       space and can be obtained, for example, from a GridFunction using the
-       Operator returned by GetElementRestriction().
-
-       The target quadrature points in the elements are described by the given
-       QuadratureSpace, @a qs. */
-   const QuadratureInterpolator *GetQuadratureInterpolator(
-      const QuadratureSpace &qs) const;
-
    /// Returns vector dimension.
    inline int GetVDim() const { return vdim; }
 
@@ -873,123 +806,6 @@ public:
    virtual const Operator &BackwardOperator();
 };
 
-
-/// Operator that converts FiniteElementSpace L-vectors to E-vectors.
-/** Objects of this type are typically created and owned by FiniteElementSpace
-    objects, see FiniteElementSpace::GetElementRestriction(). */
-class ElementRestriction : public Operator
-{
-protected:
-   const FiniteElementSpace &fes;
-   const int ne;
-   const int vdim;
-   const bool byvdim;
-   const int ndofs;
-   const int dof;
-   const int nedofs;
-   Array<int> offsets;
-   Array<int> indices;
-
-public:
-   ElementRestriction(const FiniteElementSpace&, ElementDofOrdering);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
-
-/** @brief A class that performs interpolation from an E-vector to quadrature
-    point values and/or derivatives (Q-vectors). */
-/** An E-vector represents the element-wise discontinuous version of the FE
-    space and can be obtained, for example, from a GridFunction using the
-    Operator returned by FiniteElementSpace::GetElementRestriction().
-
-    The target quadrature points in the elements can be described either by an
-    IntegrationRule (all mesh elements must be of the same type in this case) or
-    by a QuadratureSpace. */
-class QuadratureInterpolator
-{
-protected:
-   friend class FiniteElementSpace; // Needs access to qspace and IntRule
-
-   const FiniteElementSpace *fespace;  ///< Not owned
-   const QuadratureSpace *qspace;      ///< Not owned
-   const IntegrationRule *IntRule;     ///< Not owned
-
-   mutable bool use_tensor_products;
-
-   static const int MAX_NQ2D = 100;
-   static const int MAX_ND2D = 100;
-   static const int MAX_VDIM2D = 2;
-
-   static const int MAX_NQ3D = 1000;
-   static const int MAX_ND3D = 1000;
-   static const int MAX_VDIM3D = 3;
-
-public:
-   enum EvalFlags
-   {
-      VALUES       = 1 << 0,  ///< Evaluate the values at quadrature points
-      DERIVATIVES  = 1 << 1,  ///< Evaluate the derivatives at quadrature points
-      /** @brief Assuming the derivative at quadrature points form a matrix,
-          this flag can be used to compute and store their determinants. This
-          flag can only be used in Mult(). */
-      DETERMINANTS = 1 << 2
-   };
-
-   QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const IntegrationRule &ir);
-
-   QuadratureInterpolator(const FiniteElementSpace &fes,
-                          const QuadratureSpace &qs);
-
-   /** @brief Disable the use of tensor product evaluations, for tensor-product
-       elements, e.g. quads and hexes. */
-   /** Currently, tensor product evaluations are not implemented and this method
-       has no effect. */
-   void DisableTensorProducts(bool disable = true) const
-   { use_tensor_products = !disable; }
-
-   /// Interpolate the E-vector @a e_vec to quadrature points.
-   /** The @a eval_flags are a bitwise mask of constants from the EvalFlags
-       enumeration. When the VALUES flag is set, the values at quadrature points
-       are computed and stored in the Vector @a q_val. Similarly, when the flag
-       DERIVATIVES is set, the derivatives are computed and stored in @a q_der.
-       When the DETERMINANTS flags is set, it is assumed that the derivatives
-       form a matrix at each quadrature point (i.e. the associated
-       FiniteElementSpace is a vector space) and their determinants are computed
-       and stored in @a q_det. */
-   void Mult(const Vector &e_vec, unsigned eval_flags,
-             Vector &q_val, Vector &q_der, Vector &q_det) const;
-
-   /// Perform the transpose operation of Mult(). (TODO)
-   void MultTranspose(unsigned eval_flags, const Vector &q_val,
-                      const Vector &q_der, Vector &e_vec) const;
-
-   // Compute kernels follow (cannot be private or protected with nvcc)
-
-   /// Template compute kernel for 2D.
-   template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Eval2D(const int NE,
-                      const int vdim,
-                      const DofToQuad &maps,
-                      const Vector &e_vec,
-                      Vector &q_val,
-                      Vector &q_der,
-                      Vector &q_det,
-                      const int eval_flags);
-
-   /// Template compute kernel for 3D.
-   template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
-   static void Eval3D(const int NE,
-                      const int vdim,
-                      const DofToQuad &maps,
-                      const Vector &e_vec,
-                      Vector &q_val,
-                      Vector &q_der,
-                      Vector &q_det,
-                      const int eval_flags);
-};
-
 }
 
 #endif

fem/gridfunc.cpp

@@ -30,9 +30,6 @@ using namespace std;
 GridFunction::GridFunction(Mesh *m, std::istream &input)
    : Vector()
 {
-   // Grid functions are stored on the device
-   UseDevice(true);
-
    fes = new FiniteElementSpace;
    fec = fes->Load(m, input);
 
@@ -63,8 +60,6 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
 
 GridFunction::GridFunction(Mesh *m, GridFunction *gf_array[], int num_pieces)
 {
-   UseDevice(true);
-
    // all GridFunctions must have the same FE collection, vdim, ordering
    int vdim, ordering;
 
@@ -168,7 +163,6 @@ void GridFunction::Update()
       Vector old_data;
       old_data.Swap(*this);
       SetSize(T->Height());
-      UseDevice(true);
       T->Mult(old_data, *this);
    }
    else
@@ -198,9 +192,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
    MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
    if (f != fes) { Destroy(); }
    fes = f;
-   v.UseDevice(true);
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
-                    fes->GetVSize(), true);
+   NewDataAndSize((double *)v + v_offset, fes->GetVSize());
    sequence = fes->GetSequence();
 }
 
@@ -223,16 +215,13 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
    if (!f->GetProlongationMatrix())
    {
       MakeRef(f, tv, tv_offset);
-      t_vec.NewMemoryAndSize(data, size, false);
+      t_vec.NewDataAndSize(data, size);
    }
    else
    {
       MFEM_ASSERT(tv.Size() >= tv_offset + f->GetTrueVSize(), "");
       SetSpace(f); // works in parallel
-      tv.UseDevice(true);
-      const int tv_size = f->GetTrueVSize();
-      t_vec.NewMemoryAndSize(Memory<double>(tv.GetMemory(), tv_offset, tv_size),
-                             tv_size, true);
+      t_vec.NewDataAndSize(&tv(tv_offset), f->GetTrueVSize());
    }
 }
 
@@ -313,7 +302,7 @@ int GridFunction::VectorDim() const
    {
       fe = fes->GetFE(0);
    }
-   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
+   if (fe->GetRangeType() == FiniteElement::SCALAR)
    {
       return fes->GetVDim();
    }
@@ -326,7 +315,7 @@ void GridFunction::GetTrueDofs(Vector &tv) const
    if (!R)
    {
       // R is identity -> make tv a reference to *this
-      tv.NewDataAndSize(const_cast<double*>((const double*)data), size);
+      tv.NewDataAndSize(data, size);
    }
    else
    {
@@ -1378,7 +1367,7 @@ void GridFunction::AccumulateAndCountBdrValues(
          if (vdofs.Size() == 0) { continue; }
 
          transf = mesh->GetEdgeTransformation(edge);
-         transf->Attribute = -1; // TODO: set the boundary attribute
+         transf->Attribute = -1; // FIXME: set the boundary attribute
          fe = fes->GetEdgeElement(edge);
          if (!vcoeff)
          {
@@ -1482,7 +1471,7 @@ void GridFunction::AccumulateAndCountBdrTangentValues(
          if (dofs.Size() == 0) { continue; }
 
          T = mesh->GetEdgeTransformation(edge);
-         T->Attribute = -1; // TODO: set the boundary attribute
+         T->Attribute = -1; // FIXME: set the boundary attribute
          fe = fes->GetEdgeElement(edge);
          lvec.SetSize(fe->GetDof());
          fe->Project(vcoeff, *T, lvec);
@@ -1787,7 +1776,6 @@ void GridFunction::ProjectBdrCoefficient(VectorCoefficient &vcoeff,
 void GridFunction::ProjectBdrCoefficient(Coefficient *coeff[], Array<int> &attr)
 {
    Array<int> values_counter;
-   this->HostReadWrite();
    AccumulateAndCountBdrValues(coeff, NULL, attr, values_counter);
    ComputeMeans(ARITHMETIC, values_counter);
 #ifdef MFEM_DEBUG

fem/gridfunc.hpp

@@ -68,16 +68,15 @@ protected:
 
 public:
 
-   GridFunction() { fes = NULL; fec = NULL; sequence = 0; UseDevice(true); }
+   GridFunction() { fes = NULL; fec = NULL; sequence = 0; }
 
    /// Copy constructor. The internal true-dof vector #t_vec is not copied.
    GridFunction(const GridFunction &orig)
-      : Vector(orig), fes(orig.fes), fec(NULL), sequence(orig.sequence)
-   { UseDevice(true); }
+      : Vector(orig), fes(orig.fes), fec(NULL), sequence(orig.sequence) { }
 
    /// Construct a GridFunction associated with the FiniteElementSpace @a *f.
    GridFunction(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; fec = NULL; sequence = f->GetSequence(); UseDevice(true); }
+   { fes = f; fec = NULL; sequence = f->GetSequence(); }
 
    /// Construct a GridFunction using previously allocated array @a data.
    /** The GridFunction does not assume ownership of @a data which is assumed to
@@ -85,9 +84,8 @@ public:
        for externally allocated array, the pointer @a data can be NULL. The data
        array can be replaced later using the method SetData().
     */
-   GridFunction(FiniteElementSpace *f, double *data)
-      : Vector(data, f->GetVSize())
-   { fes = f; fec = NULL; sequence = f->GetSequence(); UseDevice(true); }
+   GridFunction(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
+   { fes = f; fec = NULL; sequence = f->GetSequence(); }
 
    /// Construct a GridFunction on the given Mesh, using the data from @a input.
    /** The content of @a input should be in the format created by the method
@@ -126,7 +124,6 @@ public:
 
    /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
    /// then `tv` will be set to point to the data of `*this`.
-   /** @warning This method breaks const-ness when all dofs are true. */
    void GetTrueDofs(Vector &tv) const;
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.
@@ -705,7 +702,7 @@ inline void QuadratureFunction::GetElementValues(int idx, Vector &values) const
    const int s_offset = qspace->element_offsets[idx];
    const int sl_size = qspace->element_offsets[idx+1] - s_offset;
    values.SetSize(vdim*sl_size);
-   const double *q = data + vdim*s_offset;
+   double *q = data + vdim*s_offset;
    for (int i = 0; i<values.Size(); i++)
    {
       values(i) = *(q++);
@@ -725,14 +722,12 @@ inline void QuadratureFunction::GetElementValues(int idx,
    const int s_offset = qspace->element_offsets[idx];
    const int sl_size = qspace->element_offsets[idx+1] - s_offset;
    values.SetSize(vdim, sl_size);
-   const double *q = data + vdim*s_offset;
+   double *q = data + vdim*s_offset;
    for (int j = 0; j<sl_size; j++)
-   {
       for (int i = 0; i<vdim; i++)
       {
          values(i,j) = *(q++);
       }
-   }
 }
 
 } // namespace mfem

fem/intrules.cpp

@@ -78,19 +78,6 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
    }
 }
 
-const Array<double> &IntegrationRule::GetWeights() const
-{
-   if (weights.Size() != GetNPoints())
-   {
-      weights.SetSize(GetNPoints());
-      for (int i = 0; i < GetNPoints(); i++)
-      {
-         weights[i] = IntPoint(i).weight;
-      }
-   }
-   return weights;
-}
-
 void IntegrationRule::GrundmannMollerSimplexRule(int s, int n)
 {
    // for pow on older compilers

fem/intrules.hpp

@@ -87,9 +87,6 @@ class IntegrationRule : public Array<IntegrationPoint>
 private:
    friend class IntegrationRules;
    int Order;
-   /** @brief The quadrature weights gathered as a contiguous array. Created
-       by request with the method GetWeights(). */
-   mutable Array<double> weights;
 
    /// Define n-simplex rule (triangle/tetrahedron for n=2/3) of order (2s+1)
    void GrundmannMollerSimplexRule(int s, int n = 3);
@@ -242,11 +239,6 @@ public:
    /// Returns a const reference to the i-th integration point
    const IntegrationPoint &IntPoint(int i) const { return (*this)[i]; }
 
-   /// Return the quadrature weights in a contiguous array.
-   /** If a contiguous array is not required, the weights can be accessed with
-       a call like this: `IntPoint(i).weight`. */
-   const Array<double> &GetWeights() const;
-
    /// Destroys an IntegrationRule object
    ~IntegrationRule() { }
 };

fem/linearform.cpp

@@ -19,9 +19,6 @@ namespace mfem
 LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
    : Vector(f->GetVSize())
 {
-   // Linear forms are stored on the device
-   UseDevice(true);
-
    fes = f;
    extern_lfs = 1;
 
@@ -86,10 +83,6 @@ void LinearForm::Assemble()
 
    Vector::operator=(0.0);
 
-   // The above operation is executed on device because of UseDevice().
-   // The first use of AddElementVector() below will move it back to host
-   // because both 'vdofs' and 'elemvect' are on host.
-
    if (dlfi.Size())
    {
       for (i = 0; i < fes -> GetNE(); i++)

fem/linearform.hpp

@@ -64,7 +64,7 @@ public:
    /// Creates linear form associated with FE space @a *f.
    /** The pointer @a f is not owned by the newly constructed object. */
    LinearForm(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; extern_lfs = 0; UseDevice(true); }
+   { fes = f; extern_lfs = 0; }
 
    /** @brief Create a LinearForm on the FiniteElementSpace @a f, using the
        same integrators as the LinearForm @a lf.
@@ -79,7 +79,16 @@ public:
    /** The associated FiniteElementSpace can be set later using one of the
        methods: Update(FiniteElementSpace *) or
        Update(FiniteElementSpace *, Vector &, int). */
-   LinearForm() { fes = NULL; extern_lfs = 0; UseDevice(true); }
+   LinearForm() { fes = NULL; extern_lfs = 0; }
+
+   /// Construct a LinearForm using previously allocated array @a data.
+   /** The LinearForm does not assume ownership of @a data which is assumed to
+       be of size at least `f->GetVSize()`. Similar to the Vector constructor
+       for externally allocated array, the pointer @a data can be NULL. The data
+       array can be replaced later using the method SetData().
+    */
+   LinearForm(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
+   { fes = f; }
 
    /// Copy assignment. Only the data of the base class Vector is copied.
    /** It is assumed that this object and @a rhs use FiniteElementSpace%s that

fem/lininteg.cpp

@@ -181,7 +181,7 @@ void VectorDomainLFIntegrator::AssembleRHSElementVect(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int intorder = 2*el.GetOrder();
+      int intorder = el.GetOrder() + 1;
       ir = &IntRules.Get(el.GetGeomType(), intorder);
    }
 
@@ -240,7 +240,7 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int intorder = 2*el.GetOrder();
+      int intorder = el.GetOrder() + 1;
       ir = &IntRules.Get(el.GetGeomType(), intorder);
    }
 
@@ -275,7 +275,7 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
    {
-      int intorder = 2*el.GetOrder();
+      int intorder = el.GetOrder() + 1;
       ir = &IntRules.Get(Tr.FaceGeom, intorder);
    }
 

fem/occa.okl

@@ -35,11 +35,11 @@ typedef double* QLocal2D_t @dim(Q1D, Q1D, NE);
 typedef double* DLocal3D_t @dim(D1D, D1D, D1D, NE);
 typedef double* QLocal3D_t @dim(Q1D, Q1D, Q1D, NE);
 
-typedef double* Jacobian2D_t @dim(Q2D, 2, 2, NE);
-typedef double* Jacobian3D_t @dim(Q3D, 3, 3, NE);
+typedef double* Jacobian2D_t @dim(2, 2, Q2D, NE);
+typedef double* Jacobian3D_t @dim(3, 3, Q3D, NE);
 
-typedef double* SymmOperator2D_t @dim(Q2D, 3, NE);
-typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
+typedef double* SymmOperator2D_t @dim(3, Q2D, NE);
+typedef double* SymmOperator3D_t @dim(6, Q3D, NE);
 
 @kernel void DiffusionSetup2D(const int NE,
                               @restrict const double *W,
@@ -48,12 +48,12 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
                               @restrict SymmOperator2D_t op) {
   for (int e = 0; e < NE; ++e; @outer) {
     for (int q = 0; q < Q2D; ++q; @inner) {
-      const double J11 = J(q, 0, 0, e), J12 = J(q, 1, 0, e);
-      const double J21 = J(q, 0, 1, e), J22 = J(q, 1, 1, e);
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e);
       const double c_detJ = W[q] * COEFF / ((J11 * J22) - (J21 * J12));
-      op(q, 0, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
-      op(q, 1, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
-      op(q, 2, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
+      op(0, q, e) =  c_detJ * (J21*J21 + J22*J22); // (1,1)
+      op(1, q, e) = -c_detJ * (J21*J11 + J22*J12); // (1,2), (2,1)
+      op(2, q, e) =  c_detJ * (J11*J11 + J12*J12); // (2,2)
     }
   }
 }
@@ -65,9 +65,9 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
                               @restrict SymmOperator3D_t op) {
   for (int e = 0; e < NE; ++e; @outer) {
     for (int q = 0; q < Q3D; ++q; @inner) {
-      const double J11 = J(q, 0, 0, e), J12 = J(q, 1, 0, e), J13 = J(q, 2, 0, e);
-      const double J21 = J(q, 0, 1, e), J22 = J(q, 1, 1, e), J23 = J(q, 2, 1, e);
-      const double J31 = J(q, 0, 2, e), J32 = J(q, 1, 2, e), J33 = J(q, 2, 2, e);
+      const double J11 = J(0, 0, q, e), J12 = J(1, 0, q, e), J13 = J(2, 0, q, e);
+      const double J21 = J(0, 1, q, e), J22 = J(1, 1, q, e), J23 = J(2, 1, q, e);
+      const double J31 = J(0, 2, q, e), J32 = J(1, 2, q, e), J33 = J(2, 2, q, e);
 
       const double detJ = ((J11 * J22 * J33) + (J12 * J23 * J31) + (J13 * J21 * J32) -
                            (J13 * J22 * J31) - (J12 * J21 * J33) - (J11 * J23 * J32));
@@ -88,12 +88,12 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
       const double A33 = (J11 * J22) - (J12 * J21);
 
       // adj(J)^Tadj(J)
-      op(q, 0, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
-      op(q, 1, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
-      op(q, 2, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
-      op(q, 3, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
-      op(q, 4, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
-      op(q, 5, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
+      op(0, q, e) = c_detJ * (A11*A11 + A21*A21 + A31*A31); // (1,1)
+      op(1, q, e) = c_detJ * (A11*A12 + A21*A22 + A31*A32); // (1,2), (2,1)
+      op(2, q, e) = c_detJ * (A11*A13 + A21*A23 + A31*A33); // (1,3), (3,1)
+      op(3, q, e) = c_detJ * (A12*A12 + A22*A22 + A32*A32); // (2,2)
+      op(4, q, e) = c_detJ * (A12*A13 + A22*A23 + A32*A33); // (2,3), (3,2)
+      op(5, q, e) = c_detJ * (A13*A13 + A23*A23 + A33*A33); // (3,3)
     }
   }
 }
@@ -146,9 +146,9 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
          for (int qy = 0; qy < Q1D; ++qy) {
             for (int qx = 0; qx < Q1D; ++qx) {
                const int q = QUAD_2D_ID(qx, qy);
-               const double O11 = op(q, 0, e);
-               const double O12 = op(q, 1, e);
-               const double O22 = op(q, 2, e);
+               const double O11 = op(0, q, e);
+               const double O12 = op(1, q, e);
+               const double O22 = op(2, q, e);
 
                const double gradX = grad[qy][qx][0];
                const double gradY = grad[qy][qx][1];
@@ -255,9 +255,9 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
                      }
 
                      const int q = QUAD_2D_ID(qx, qy);
-                     const double O11 = op(q, 0, e);
-                     const double O12 = op(q, 1, e);
-                     const double O22 = op(q, 2, e);
+                     const double O11 = op(0, q, e);
+                     const double O12 = op(1, q, e);
+                     const double O22 = op(2, q, e);
 
                      s_grad(0, qx, qy) = (O11 * gradX) + (O12 * gradY);
                      s_grad(1, qx, qy) = (O12 * gradX) + (O22 * gradY);
@@ -382,12 +382,12 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
         for (int qy = 0; qy < Q1D; ++qy) {
           for (int qx = 0; qx < Q1D; ++qx) {
             const int q = QUAD_3D_ID(qx, qy, qz);
-            const double O11 = op(q, 0, e);
-            const double O12 = op(q, 1, e);
-            const double O13 = op(q, 2, e);
-            const double O22 = op(q, 3, e);
-            const double O23 = op(q, 4, e);
-            const double O33 = op(q, 5, e);
+            const double O11 = op(0, q, e);
+            const double O12 = op(1, q, e);
+            const double O13 = op(2, q, e);
+            const double O22 = op(3, q, e);
+            const double O23 = op(4, q, e);
+            const double O33 = op(5, q, e);
 
             const double gradX = grad[qz][qy][qx][0];
             const double gradY = grad[qz][qy][qx][1];
@@ -557,12 +557,12 @@ typedef double* SymmOperator3D_t @dim(Q3D, 6, NE);
             }
 
             const int q = QUAD_3D_ID(qx, qy, qz);
-            const double O11 = op(q, 0, e);
-            const double O12 = op(q, 1, e);
-            const double O13 = op(q, 2, e);
-            const double O22 = op(q, 3, e);
-            const double O23 = op(q, 4, e);
-            const double O33 = op(q, 5, e);
+            const double O11 = op(0, q, e);
+            const double O12 = op(1, q, e);
+            const double O13 = op(2, q, e);
+            const double O22 = op(3, q, e);
+            const double O23 = op(4, q, e);
+            const double O33 = op(5, q, e);
 
             const double qDxyz = (O11 * Dxyz) + (O12 * xDyz) + (O13 * xyDz);
             const double qxDyz = (O12 * Dxyz) + (O22 * xDyz) + (O23 * xyDz);

fem/pbilinearform.cpp

@@ -203,14 +203,7 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
       vdofs1.Copy(vdofs_all);
       for (int j = 0; j < vdofs2.Size(); j++)
       {
-         if (vdofs2[j] >= 0)
-         {
-            vdofs2[j] += height;
-         }
-         else
-         {
-            vdofs2[j] -= height;
-         }
+         vdofs2[j] += height;
       }
       vdofs_all.Append(vdofs2);
       for (int k = 0; k < fbfi.Size(); k++)

fem/pfespace.cpp

@@ -613,15 +613,15 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
    int ldof  = GetVSize();
    int ltdof = TrueVSize();
 
-   HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
-   HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
+   HYPRE_Int *i_diag = mfem::New<HYPRE_Int>(ldof+1);
+   HYPRE_Int *j_diag = mfem::New<HYPRE_Int>(ltdof);
    int diag_counter;
 
-   HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
-   HYPRE_Int *j_offd = new HYPRE_Int[ldof-ltdof];
+   HYPRE_Int *i_offd = mfem::New<HYPRE_Int>(ldof+1);
+   HYPRE_Int *j_offd = mfem::New<HYPRE_Int>(ldof-ltdof);
    int offd_counter;
 
-   HYPRE_Int *cmap   = new HYPRE_Int[ldof-ltdof];
+   HYPRE_Int *cmap   = mfem::New<HYPRE_Int>(ldof-ltdof);
 
    HYPRE_Int *col_starts = GetTrueDofOffsets();
    HYPRE_Int *row_starts = GetDofOffsets();
@@ -747,14 +747,12 @@ void ParFiniteElementSpace::GetEssentialTrueDofs(const Array<int>
    // Verify that in boolean arithmetic: P^T ess_dofs = R ess_dofs.
    Array<int> true_ess_dofs2(true_ess_dofs.Size());
    HypreParMatrix *Pt = Dof_TrueDof_Matrix()->Transpose();
-   const int *ess_dofs_data = ess_dofs.HostRead();
-   Pt->BooleanMult(1, ess_dofs_data, 0, true_ess_dofs2);
+   Pt->BooleanMult(1, ess_dofs, 0, true_ess_dofs2);
    delete Pt;
    int counter = 0;
-   const int *ted = true_ess_dofs.HostRead();
    for (int i = 0; i < true_ess_dofs.Size(); i++)
    {
-      if (bool(ted[i]) != bool(true_ess_dofs2[i])) { counter++; }
+      if (bool(true_ess_dofs[i]) != bool(true_ess_dofs2[i])) { counter++; }
    }
    MFEM_VERIFY(counter == 0, "internal MFEM error: counter = " << counter);
 #endif
@@ -904,15 +902,11 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
       {
          GetElementVDofs(my_elems[i], ldofs);
          for (int j = 0; j < ldofs.Size(); j++)
-         {
-            int ldof = (ldofs[j] >= 0 ? ldofs[j] : -1-ldofs[j]);
-
-            if (ldof_marker[ldof] != fn)
+            if (ldof_marker[ldofs[j]] != fn)
             {
-               ldof_marker[ldof] = fn;
+               ldof_marker[ldofs[j]] = fn;
                send_face_nbr_ldof.AddAColumnInRow(fn);
             }
-         }
          send_nbr_elem_dof.AddColumnsInRow(send_el_off[fn] + i, ldofs.Size());
       }
 
@@ -966,11 +960,9 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
          GetElementVDofs(my_elems[i], ldofs);
          for (int j = 0; j < ldofs.Size(); j++)
          {
-            int ldof = (ldofs[j] >= 0 ? ldofs[j] : -1-ldofs[j]);
-
-            if (ldof_marker[ldof] != fn)
+            if (ldof_marker[ldofs[j]] != fn)
             {
-               ldof_marker[ldof] = fn;
+               ldof_marker[ldofs[j]] = fn;
                send_face_nbr_ldof.AddConnection(fn, ldofs[j]);
             }
          }
@@ -991,14 +983,12 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
 
       for (int i = 0; i < num_ldofs; i++)
       {
-         int ldof = (ldofs[i] >= 0 ? ldofs[i] : -1-ldofs[i]);
-         ldof_marker[ldof] = i;
+         ldof_marker[ldofs[i]] = i;
       }
 
       for ( ; j < j_end; j++)
       {
-         int ldof = (send_J[j] >= 0 ? send_J[j] : -1-send_J[j]);
-         send_J[j] = (send_J[j] >= 0 ? ldof_marker[ldof] : -1-ldof_marker[ldof]);
+         send_J[j] = ldof_marker[send_J[j]];
       }
    }
 
@@ -1033,14 +1023,7 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
 
       for ( ; j < j_end; j++)
       {
-         if (recv_J[j] >= 0)
-         {
-            recv_J[j] += shift;
-         }
-         else
-         {
-            recv_J[j] -= shift;
-         }
+         recv_J[j] += shift;
       }
    }
 
@@ -1089,15 +1072,8 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
    for (int fn = 0, j = 0; fn < num_face_nbrs; fn++)
    {
       for (int j_end = face_nbr_ldof.GetI()[fn+1]; j < j_end; j++)
-      {
-         int ldof = face_nbr_ldof.GetJ()[j];
-         if (ldof < 0)
-         {
-            ldof = -1-ldof;
-         }
-
-         face_nbr_glob_dof_map[j] = dof_face_nbr_offsets[fn] + ldof;
-      }
+         face_nbr_glob_dof_map[j] =
+            dof_face_nbr_offsets[fn] + face_nbr_ldof.GetJ()[j];
    }
 
    MPI_Waitall(num_face_nbrs, send_requests, statuses);
@@ -2273,7 +2249,7 @@ HypreParMatrix* ParFiniteElementSpace
    }
 
    // create offd column mapping
-   HYPRE_Int *cmap = new HYPRE_Int[col_map.size()];
+   HYPRE_Int *cmap = mfem::New<HYPRE_Int>(col_map.size());
    int offd_col = 0;
    for (std::map<HYPRE_Int, int>::iterator
         it = col_map.begin(); it != col_map.end(); ++it)
@@ -2282,14 +2258,14 @@ HypreParMatrix* ParFiniteElementSpace
       it->second = offd_col++;
    }
 
-   HYPRE_Int *I_diag = new HYPRE_Int[vdim*local_rows + 1];
-   HYPRE_Int *I_offd = new HYPRE_Int[vdim*local_rows + 1];
+   HYPRE_Int *I_diag = mfem::New<HYPRE_Int>(vdim*local_rows + 1);
+   HYPRE_Int *I_offd = mfem::New<HYPRE_Int>(vdim*local_rows + 1);
 
-   HYPRE_Int *J_diag = new HYPRE_Int[nnz_diag];
-   HYPRE_Int *J_offd = new HYPRE_Int[nnz_offd];
+   HYPRE_Int *J_diag = mfem::New<HYPRE_Int>(nnz_diag);
+   HYPRE_Int *J_offd = mfem::New<HYPRE_Int>(nnz_offd);
 
-   double *A_diag = new double[nnz_diag];
-   double *A_offd = new double[nnz_offd];
+   double *A_diag = mfem::New<double>(nnz_diag);
+   double *A_offd = mfem::New<double>(nnz_offd);
 
    int vdim1 = bynodes ? vdim : 1;
    int vdim2 = bynodes ? 1 : vdim;
@@ -2340,7 +2316,7 @@ HypreParMatrix* ParFiniteElementSpace
 
 static HYPRE_Int* make_i_array(int nrows)
 {
-   HYPRE_Int *I = new HYPRE_Int[nrows+1];
+   HYPRE_Int *I = mfem::New<HYPRE_Int>(nrows+1);
    for (int i = 0; i <= nrows; i++) { I[i] = -1; }
    return I;
 }
@@ -2352,7 +2328,7 @@ static HYPRE_Int* make_j_array(HYPRE_Int* I, int nrows)
    {
       if (I[i] >= 0) { nnz++; }
    }
-   HYPRE_Int *J = new HYPRE_Int[nnz];
+   HYPRE_Int *J = mfem::New<HYPRE_Int>(nnz);
 
    I[nrows] = -1;
    for (int i = 0, k = 0; i <= nrows; i++)
@@ -2451,7 +2427,7 @@ ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
    }
    SortPairs<HYPRE_Int, int>(cmap_offd, offd_cols);
 
-   HYPRE_Int* cmap = new HYPRE_Int[offd_cols];
+   HYPRE_Int* cmap = mfem::New<HYPRE_Int>(offd_cols);
    for (int i = 0; i < offd_cols; i++)
    {
       cmap[i] = cmap_offd[i].one;
@@ -2647,7 +2623,7 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
    offd->SetWidth(col_map.size());
 
    // create offd column mapping for use by hypre
-   HYPRE_Int *cmap = new HYPRE_Int[offd->Width()];
+   HYPRE_Int *cmap = mfem::New<HYPRE_Int>(offd->Width());
    for (std::map<HYPRE_Int, int>::iterator
         it = col_map.begin(); it != col_map.end(); ++it)
    {
@@ -2887,8 +2863,9 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
    MFEM_ASSERT(x.Size() == Width(), "");
    MFEM_ASSERT(y.Size() == Height(), "");
 
-   const double *xdata = x.HostRead();
-   double *ydata = y.HostWrite();
+   const double *xdata = x.GetData();
+   double *ydata = y.GetData();
+   x.Pull();
    const int m = external_ldofs.Size();
 
    const int in_layout = 2; // 2 - input is ltdofs array
@@ -2905,6 +2882,7 @@ void ConformingProlongationOperator::Mult(const Vector &x, Vector &y) const
 
    const int out_layout = 0; // 0 - output is ldofs array
    gc.BcastEnd(ydata, out_layout);
+   y.Push();
 }
 
 void ConformingProlongationOperator::MultTranspose(
@@ -2913,8 +2891,9 @@ void ConformingProlongationOperator::MultTranspose(
    MFEM_ASSERT(x.Size() == Height(), "");
    MFEM_ASSERT(y.Size() == Width(), "");
 
-   const double *xdata = x.HostRead();
-   double *ydata = y.HostWrite();
+   const double *xdata = x.GetData();
+   double *ydata = y.GetData();
+   x.Pull();
    const int m = external_ldofs.Size();
 
    gc.ReduceBegin(xdata);
@@ -2930,6 +2909,7 @@ void ConformingProlongationOperator::MultTranspose(
 
    const int out_layout = 2; // 2 - output is an array on all ltdofs
    gc.ReduceEnd<double>(ydata, out_layout, GroupCommunicator::Sum);
+   y.Push();
 }
 
 } // namespace mfem

fem/pgridfunc.cpp

@@ -367,10 +367,10 @@ void ParGridFunction::ProjectDiscCoefficient(Coefficient &coeff, AvgType type)
    GroupCommunicator &gcomm = pfes->GroupComm();
    gcomm.Reduce<int>(zones_per_vdof, GroupCommunicator::Sum);
    gcomm.Bcast(zones_per_vdof);
-
-   // Accumulate for all vdofs.
-   gcomm.Reduce<double>(data, GroupCommunicator::Sum);
-   gcomm.Bcast<double>(data);
+   // Accumulate for all tdofs.
+   HypreParVector *tv = this->ParallelAssemble();
+   this->Distribute(tv);
+   delete tv;
 
    ComputeMeans(type, zones_per_vdof);
 }
@@ -389,10 +389,10 @@ void ParGridFunction::ProjectDiscCoefficient(VectorCoefficient &vcoeff,
    GroupCommunicator &gcomm = pfes->GroupComm();
    gcomm.Reduce<int>(zones_per_vdof, GroupCommunicator::Sum);
    gcomm.Bcast(zones_per_vdof);
-
-   // Accumulate for all vdofs.
-   gcomm.Reduce<double>(data, GroupCommunicator::Sum);
-   gcomm.Bcast<double>(data);
+   // Accumulate for all tdofs.
+   HypreParVector *tv = this->ParallelAssemble();
+   this->Distribute(tv);
+   delete tv;
 
    ComputeMeans(type, zones_per_vdof);
 }
@@ -425,8 +425,8 @@ void ParGridFunction::ProjectBdrCoefficient(
    }
    else
    {
-      // TODO: is this the same as the conforming case (after the merge of
-      //       cut-mesh-groups-dev)?
+      // FIXME: same as the conforming case after 'cut-mesh-groups-dev-*' is
+      //        merged?
       ComputeMeans(ARITHMETIC, values_counter);
    }
 #ifdef MFEM_DEBUG
@@ -469,8 +469,8 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
    }
    else
    {
-      // TODO: is this the same as the conforming case (after the merge of
-      //       cut-mesh-groups-dev)?
+      // FIXME: same as the conforming case after 'cut-mesh-groups-dev-*' is
+      //        merged?
       ComputeMeans(ARITHMETIC, values_counter);
    }
 #ifdef MFEM_DEBUG
@@ -487,17 +487,16 @@ void ParGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
 
 void ParGridFunction::Save(std::ostream &out) const
 {
-   double *data_  = const_cast<double*>(HostRead());
    for (int i = 0; i < size; i++)
    {
-      if (pfes->GetDofSign(i) < 0) { data_[i] = -data_[i]; }
+      if (pfes->GetDofSign(i) < 0) { data[i] = -data[i]; }
    }
 
    GridFunction::Save(out);
 
    for (int i = 0; i < size; i++)
    {
-      if (pfes->GetDofSign(i) < 0) { data_[i] = -data_[i]; }
+      if (pfes->GetDofSign(i) < 0) { data[i] = -data[i]; }
    }
 }
 

fem/plinearform.hpp

@@ -45,6 +45,16 @@ public:
    /** The pointer @a pf is not owned by the newly constructed object. */
    ParLinearForm(ParFiniteElementSpace *pf) : LinearForm(pf) { pfes = pf; }
 
+   /// Construct a ParLinearForm using previously allocated array @a data.
+   /** The ParLinearForm does not assume ownership of @a data which is assumed
+       to be of size at least `pf->GetVSize()`. Similar to the LinearForm and
+       Vector constructors for externally allocated array, the pointer @a data
+       can be NULL. The data array can be replaced later using the method
+       SetData().
+    */
+   ParLinearForm(ParFiniteElementSpace *pf, double *data) :
+      LinearForm(pf, data), pfes(pf) { }
+
    /** @brief Create a ParLinearForm on the ParFiniteElementSpace @a *pf, using
        the same integrators as the ParLinearForm @a *plf.
 

fem/tmop.cpp

@@ -956,13 +956,13 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
       Tpr->Attribute = T.Attribute;
       Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
    }
-   // TODO: computing the coefficients 'coeff1' and 'coeff0' in physical
-   //       coordinates means that, generally, the gradient and Hessian of the
-   //       TMOP_Integrator will depend on the derivatives of the coefficients.
+   // FIXME: computing the coefficients 'coeff1' and 'coeff0' in physical
+   //        coordinates means that, generally, the gradient and Hessian of the
+   //        TMOP_Integrator will depend on the derivatives of the coefficients.
    //
-   //       In some cases the coefficients are independent of any movement of
-   //       the physical coordinates (i.e. changes in 'elfun'), e.g. when the
-   //       coefficient is a ConstantCoefficient or a GridFunctionCoefficient.
+   //        In some cases the coefficients are independent of any movement of
+   //        the physical coordinates (i.e. changes in 'elfun'), e.g. when the
+   //        coefficient is a ConstantCoefficient or a GridFunctionCoefficient.
 
    for (int i = 0; i < ir->GetNPoints(); i++)
    {

general/array.cpp

@@ -19,12 +19,54 @@
 namespace mfem
 {
 
+BaseArray::BaseArray(int asize, int ainc, int elementsize)
+{
+   if (asize > 0)
+   {
+      data = mfem::New<char>(asize * elementsize);
+      size = allocsize = asize;
+   }
+   else
+   {
+      data = 0;
+      size = allocsize = 0;
+   }
+   inc = ainc;
+}
+
+BaseArray::~BaseArray()
+{
+   if (allocsize > 0)
+   {
+      mfem::Delete((char*)data);
+   }
+}
+
+void BaseArray::GrowSize(int minsize, int elementsize)
+{
+   void *p;
+   int nsize = (inc > 0) ? abs(allocsize) + inc : 2 * abs(allocsize);
+   if (nsize < minsize) { nsize = minsize; }
+
+   p = mfem::New<char>(nsize * elementsize);
+   if (size > 0)
+   {
+      mfem::Memcpy(p, data, size * elementsize);
+   }
+   if (allocsize > 0)
+   {
+      mfem::Delete((char*)data);
+   }
+   data = p;
+   allocsize = nsize;
+}
+
 template <class T>
 void Array<T>::Print(std::ostream &out, int width) const
 {
    for (int i = 0; i < size; i++)
    {
-      out << data[i];
+      out << ((T*)data)[i];
       if ( !((i+1) % width) || i+1 == size )
       {
          out << '\n';
@@ -71,12 +113,10 @@ T Array<T>::Max() const
 
    T max = operator[](0);
    for (int i = 1; i < size; i++)
-   {
       if (max < operator[](i))
       {
          max = operator[](i);
       }
-   }
 
    return max;
 }
@@ -88,12 +128,10 @@ T Array<T>::Min() const
 
    T min = operator[](0);
    for (int i = 1; i < size; i++)
-   {
       if (operator[](i) < min)
       {
          min = operator[](i);
       }
-   }
 
    return min;
 }

general/array.hpp

@@ -14,7 +14,6 @@
 
 #include "../config/config.hpp"
 #include "mem_manager.hpp"
-#include "device.hpp"
 #include "error.hpp"
 #include "globals.hpp"
 
@@ -26,6 +25,31 @@
 namespace mfem
 {
 
+/// Base class for array container.
+class BaseArray
+{
+protected:
+   /// Pointer to data
+   void *data;
+   /// Size of the array
+   int size;
+   /// Size of the allocated memory
+   int allocsize;
+   /** Increment of allocated memory on overflow,
+       inc = 0 doubles the array */
+   int inc;
+
+   BaseArray() { }
+   /// Creates array of asize elements of size elementsize
+   BaseArray(int asize, int ainc, int elmentsize);
+   /// Free the allocated memory
+   ~BaseArray();
+   /** Increases the allocsize of the array to be at least minsize.
+       The current content of the array is copied to the newly allocated
+       space. minsize must be > abs(allocsize). */
+   void GrowSize(int minsize, int elementsize);
+};
+
 template <class T>
 class Array;
 
@@ -41,81 +65,70 @@ void Swap(Array<T> &, Array<T> &);
    The elements can be accessed by the [] operator, the range is 0 to size-1.
 */
 template <class T>
-class Array
+class Array : public BaseArray
 {
-protected:
-   /// Pointer to data
-   Memory<T> data;
-   /// Size of the array
-   int size;
-
-   inline void GrowSize(int minsize);
-
 public:
    friend void Swap<T>(Array<T> &, Array<T> &);
 
-   /// Creates an empty array
-   inline Array() : size(0) { data.Reset(); }
-
    /// Creates array of asize elements
-   explicit inline Array(int asize)
-      : size(asize) { asize > 0 ? data.New(asize) : data.Reset(); }
+   explicit inline Array(int asize = 0, int ainc = 0)
+      : BaseArray(asize, ainc, sizeof (T)) { }
 
    /** Creates array using an existing c-array of asize elements;
        allocsize is set to -asize to indicate that the data will not
        be deleted. */
-   inline Array(T *_data, int asize)
-   { data.Wrap(_data, asize, false); size = asize; }
+   inline Array(T *_data, int asize, int ainc = 0)
+   { data = _data; size = asize; allocsize = -asize; inc = ainc; }
 
    /// Copy constructor: deep copy
-   /** This method supports source arrays using any MemoryType. */
-   inline Array(const Array &src);
+   Array(const Array<T> &src)
+      : BaseArray(src.size, 0, sizeof(T))
+   { mfem::Memcpy(data, src.data, size*sizeof(T)); }
 
    /// Copy constructor (deep copy) from an Array of convertable type
    template <typename CT>
-   inline Array(const Array<CT> &src);
+   Array(const Array<CT> &src)
+      : BaseArray(src.Size(), 0, sizeof(T))
+   { for (int i = 0; i < size; i++) { (*this)[i] = T(src[i]); } }
 
    /// Destructor
-   inline ~Array() { data.Delete(); }
+   inline ~Array() { }
 
    /// Assignment operator: deep copy
    Array<T> &operator=(const Array<T> &src) { src.Copy(*this); return *this; }
 
    /// Assignment operator (deep copy) from an Array of convertable type
    template <typename CT>
-   inline Array &operator=(const Array<CT> &src);
+   Array<T> &operator=(const Array<CT> &src)
+   {
+      SetSize(src.Size());
+      for (int i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
+      return *this;
+   }
 
    /// Return the data as 'T *'
-   inline operator T *() { return data; }
+   inline operator T *() { return (T *)data; }
 
    /// Return the data as 'const T *'
-   inline operator const T *() const { return data; }
+   inline operator const T *() const { return (const T *)data; }
 
    /// Returns the data
-   inline T *GetData() { return data; }
+   inline T *GetData() { return (T *)data; }
    /// Returns the data
-   inline const T *GetData() const { return data; }
-
-   /// Return a reference to the Memory object used by the Array.
-   Memory<T> &GetMemory() { return data; }
-
-   /// Return a reference to the Memory object used by the Array, const version.
-   const Memory<T> &GetMemory() const { return data; }
-
-   /// Return the device flag of the Memory object used by the Array
-   bool UseDevice() const { return data.UseDevice(); }
+   inline const T *GetData() const { return (T *)data; }
 
    /// Return true if the data will be deleted by the array
-   inline bool OwnsData() const { return data.OwnsHostPtr(); }
+   inline bool OwnsData() const { return (allocsize > 0); }
 
    /// Changes the ownership of the data
-   inline void StealData(T **p) { *p = data; data.Reset(); size = 0; }
+   inline void StealData(T **p)
+   { *p = (T*)data; data = 0; size = allocsize = 0; }
 
    /// NULL-ifies the data
-   inline void LoseData() { data.Reset(); size = 0; }
+   inline void LoseData() { data = 0; size = allocsize = 0; }
 
    /// Make the Array own the data
-   void MakeDataOwner() const { data.SetHostPtrOwner(true); }
+   void MakeDataOwner() { allocsize = abs(allocsize); }
 
    /// Logical size of the array
    inline int Size() const { return size; }
@@ -126,18 +139,13 @@ public:
    /// Same as SetSize(int) plus initialize new entries with 'initval'
    inline void SetSize(int nsize, const T &initval);
 
-   /** @brief Resize the array to size @a nsize using MemoryType @a mt. Note
-       that unlike the other versions of SetSize(), the current content of the
-       array is not preserved. */
-   inline void SetSize(int nsize, MemoryType mt);
-
    /** Maximum number of entries the array can store without allocating more
        memory. */
-   inline int Capacity() const { return data.Capacity(); }
+   inline int Capacity() const { return abs(allocsize); }
 
    /// Ensures that the allocated size is at least the given size.
    inline void Reserve(int capacity)
-   { if (capacity > Capacity()) { GrowSize(capacity); } }
+   { if (capacity > abs(allocsize)) { GrowSize(capacity, sizeof(T)); } }
 
    /// Access element
    inline T & operator[](int i);
@@ -180,7 +188,11 @@ public:
    inline void DeleteAll();
 
    /// Create a copy of the current array
-   inline void Copy(Array &copy) const;
+   inline void Copy(Array &copy) const
+   {
+      copy.SetSize(Size());
+      mfem::Memcpy(copy.GetData(), data, Size()*sizeof(T));
+   }
 
    /// Make this Array a reference to a pointer
    inline void MakeRef(T *, int);
@@ -188,7 +200,7 @@ public:
    /// Make this Array a reference to 'master'
    inline void MakeRef(const Array &master);
 
-   inline void GetSubArray(int offset, int sa_size, Array<T> &sa) const;
+   inline void GetSubArray(int offset, int sa_size, Array<T> &sa);
 
    /// Prints array to stream with width elements per row
    void Print(std::ostream &out = mfem::out, int width = 4) const;
@@ -223,18 +235,18 @@ public:
    T Min() const;
 
    /// Sorts the array. This requires operator< to be defined for T.
-   void Sort() { std::sort((T*)data, data + size); }
+   void Sort() { std::sort((T*) data, (T*) data + size); }
 
    /// Sorts the array using the supplied comparison function object.
    template<class Compare>
-   void Sort(Compare cmp) { std::sort((T*)data, data + size, cmp); }
+   void Sort(Compare cmp) { std::sort((T*) data, (T*) data + size, cmp); }
 
    /** Removes duplicities from a sorted array. This requires operator== to be
        defined for T. */
    void Unique()
    {
-      T* end = std::unique((T*)data, data + size);
-      SetSize(end - data);
+      T* end = std::unique((T*) data, (T*) data + size);
+      SetSize(end - (T*) data);
    }
 
    /// return true if the array is sorted.
@@ -255,36 +267,10 @@ public:
    inline void CopyTo(U *dest) { std::copy(begin(), end(), dest); }
 
    // STL-like begin/end
-   inline T* begin() { return data; }
-   inline T* end() { return data + size; }
-   inline const T* begin() const { return data; }
-   inline const T* end() const { return data + size; }
+   inline T* begin() const { return (T*) data; }
+   inline T* end() const { return (T*) data + size; }
 
    long MemoryUsage() const { return Capacity() * sizeof(T); }
-
-   /// Shortcut for mfem::Read(a.GetMemory(), a.Size(), on_dev).
-   const T *Read(bool on_dev = true) const
-   { return mfem::Read(data, size, on_dev); }
-
-   /// Shortcut for mfem::Read(a.GetMemory(), a.Size(), false).
-   const T *HostRead() const
-   { return mfem::Read(data, size, false); }
-
-   /// Shortcut for mfem::Write(a.GetMemory(), a.Size(), on_dev).
-   T *Write(bool on_dev = true)
-   { return mfem::Write(data, size, on_dev); }
-
-   /// Shortcut for mfem::Write(a.GetMemory(), a.Size(), false).
-   T *HostWrite()
-   { return mfem::Write(data, size, false); }
-
-   /// Shortcut for mfem::ReadWrite(a.GetMemory(), a.Size(), on_dev).
-   T *ReadWrite(bool on_dev = true)
-   { return mfem::ReadWrite(data, size, on_dev); }
-
-   /// Shortcut for mfem::ReadWrite(a.GetMemory(), a.Size(), false).
-   T *HostReadWrite()
-   { return mfem::ReadWrite(data, size, false); }
 };
 
 template <class T>
@@ -292,9 +278,7 @@ inline bool operator==(const Array<T> &LHS, const Array<T> &RHS)
 {
    if ( LHS.Size() != RHS.Size() ) { return false; }
    for (int i=0; i<LHS.Size(); i++)
-   {
       if ( LHS[i] != RHS[i] ) { return false; }
-   }
    return true;
 }
 
@@ -581,51 +565,17 @@ inline void Swap(Array<T> &a, Array<T> &b)
 {
    Swap(a.data, b.data);
    Swap(a.size, b.size);
-}
-
-template <class T>
-inline Array<T>::Array(const Array &src)
-   : size(src.Size())
-{
-   size > 0 ? data.New(size, src.data.GetMemoryType()) : data.Reset();
-   data.CopyFrom(src.data, size);
-   data.UseDevice(src.data.UseDevice());
-}
-
-template <typename T> template <typename CT>
-inline Array<T>::Array(const Array<CT> &src)
-   : size(src.Size())
-{
-   size > 0 ? data.New(size) : data.Reset();
-   for (int i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
-}
-
-template <class T>
-inline void Array<T>::GrowSize(int minsize)
-{
-   const int nsize = std::max(minsize, 2 * data.Capacity());
-   Memory<T> p(nsize, data.GetMemoryType());
-   p.CopyFrom(data, size);
-   p.UseDevice(data.UseDevice());
-   data.Delete();
-   data = p;
-}
-
-template <typename T> template <typename CT>
-inline Array<T> &Array<T>::operator=(const Array<CT> &src)
-{
-   SetSize(src.Size());
-   for (int i = 0; i < size; i++) { (*this)[i] = T(src[i]); }
-   return *this;
+   Swap(a.allocsize, b.allocsize);
+   Swap(a.inc, b.inc);
 }
 
 template <class T>
 inline void Array<T>::SetSize(int nsize)
 {
    MFEM_ASSERT( nsize>=0, "Size must be non-negative.  It is " << nsize );
-   if (nsize > Capacity())
+   if (nsize > abs(allocsize))
    {
-      GrowSize(nsize);
+      GrowSize(nsize, sizeof(T));
    }
    size = nsize;
 }
@@ -636,51 +586,24 @@ inline void Array<T>::SetSize(int nsize, const T &initval)
    MFEM_ASSERT( nsize>=0, "Size must be non-negative.  It is " << nsize );
    if (nsize > size)
    {
-      if (nsize > Capacity())
+      if (nsize > abs(allocsize))
       {
-         GrowSize(nsize);
+         GrowSize(nsize, sizeof(T));
       }
       for (int i = size; i < nsize; i++)
       {
-         data[i] = initval;
+         ((T*)data)[i] = initval;
       }
    }
    size = nsize;
 }
 
-template <class T>
-inline void Array<T>::SetSize(int nsize, MemoryType mt)
-{
-   MFEM_ASSERT(nsize >= 0, "invalid new size: " << nsize);
-   if (mt == data.GetMemoryType())
-   {
-      if (nsize <= Capacity())
-      {
-         size = nsize;
-         return;
-      }
-   }
-   const bool use_dev = data.UseDevice();
-   data.Delete();
-   if (nsize > 0)
-   {
-      data.New(nsize, mt);
-      size = nsize;
-   }
-   else
-   {
-      data.Reset();
-      size = 0;
-   }
-   data.UseDevice(use_dev);
-}
-
 template <class T>
 inline T &Array<T>::operator[](int i)
 {
    MFEM_ASSERT( i>=0 && i<size,
                 "Access element " << i << " of array, size = " << size );
-   return data[i];
+   return ((T*)data)[i];
 }
 
 template <class T>
@@ -688,14 +611,14 @@ inline const T &Array<T>::operator[](int i) const
 {
    MFEM_ASSERT( i>=0 && i<size,
                 "Access element " << i << " of array, size = " << size );
-   return data[i];
+   return ((T*)data)[i];
 }
 
 template <class T>
 inline int Array<T>::Append(const T &el)
 {
    SetSize(size+1);
-   data[size-1] = el;
+   ((T*)data)[size-1] = el;
    return size;
 }
 
@@ -707,7 +630,7 @@ inline int Array<T>::Append(const T *els, int nels)
    SetSize(size + nels);
    for (int i = 0; i < nels; i++)
    {
-      data[old_size+i] = els[i];
+      ((T*)data)[old_size+i] = els[i];
    }
    return size;
 }
@@ -718,9 +641,9 @@ inline int Array<T>::Prepend(const T &el)
    SetSize(size+1);
    for (int i = size-1; i > 0; i--)
    {
-      data[i] = data[i-1];
+      ((T*)data)[i] = ((T*)data)[i-1];
    }
-   data[0] = el;
+   ((T*)data)[0] = el;
    return size;
 }
 
@@ -728,21 +651,21 @@ template <class T>
 inline T &Array<T>::Last()
 {
    MFEM_ASSERT(size > 0, "Array size is zero: " << size);
-   return data[size-1];
+   return ((T*)data)[size-1];
 }
 
 template <class T>
 inline const T &Array<T>::Last() const
 {
    MFEM_ASSERT(size > 0, "Array size is zero: " << size);
-   return data[size-1];
+   return ((T*)data)[size-1];
 }
 
 template <class T>
 inline int Array<T>::Union(const T &el)
 {
    int i = 0;
-   while ((i < size) && (data[i] != el)) { i++; }
+   while ((i < size) && (((T*)data)[i] != el)) { i++; }
    if (i == size)
    {
       Append(el);
@@ -755,7 +678,7 @@ inline int Array<T>::Find(const T &el) const
 {
    for (int i = 0; i < size; i++)
    {
-      if (data[i] == el) { return i; }
+      if (((T*)data)[i] == el) { return i; }
    }
    return -1;
 }
@@ -763,7 +686,7 @@ inline int Array<T>::Find(const T &el) const
 template <class T>
 inline int Array<T>::FindSorted(const T &el) const
 {
-   const T *begin = data, *end = begin + size;
+   const T *begin = (const T*) data, *end = begin + size;
    const T* first = std::lower_bound(begin, end, el);
    if (first == end || !(*first == el)) { return  -1; }
    return first - begin;
@@ -774,11 +697,11 @@ inline void Array<T>::DeleteFirst(const T &el)
 {
    for (int i = 0; i < size; i++)
    {
-      if (data[i] == el)
+      if (((T*)data)[i] == el)
       {
          for (i++; i < size; i++)
          {
-            data[i-1] = data[i];
+            ((T*)data)[i-1] = ((T*)data)[i];
          }
          size--;
          return;
@@ -789,40 +712,41 @@ inline void Array<T>::DeleteFirst(const T &el)
 template <class T>
 inline void Array<T>::DeleteAll()
 {
-   const bool use_dev = data.UseDevice();
-   data.Delete();
-   data.Reset();
-   size = 0;
-   data.UseDevice(use_dev);
-}
-
-template <typename T>
-inline void Array<T>::Copy(Array &copy) const
-{
-   copy.SetSize(Size(), data.GetMemoryType());
-   data.CopyTo(copy.data, Size());
-   copy.data.UseDevice(data.UseDevice());
+   if (allocsize > 0)
+   {
+      mfem::Delete((char*)data);
+   }
+   data = NULL;
+   size = allocsize = 0;
 }
 
 template <class T>
 inline void Array<T>::MakeRef(T *p, int s)
 {
-   data.Delete();
-   data.Wrap(p, s, false);
+   if (allocsize > 0)
+   {
+      mfem::Delete((char*)data);
+   }
+   data = p;
    size = s;
+   allocsize = -s;
 }
 
 template <class T>
 inline void Array<T>::MakeRef(const Array &master)
 {
-   data.Delete();
-   data = master.data; // note: copies the device flag
+   if (allocsize > 0)
+   {
+      mfem::Delete((char*)data);
+   }
+   data = master.data;
    size = master.size;
-   data.ClearOwnerFlags();
+   allocsize = -abs(master.allocsize);
+   inc = master.inc;
 }
 
 template <class T>
-inline void Array<T>::GetSubArray(int offset, int sa_size, Array<T> &sa) const
+inline void Array<T>::GetSubArray(int offset, int sa_size, Array<T> &sa)
 {
    sa.SetSize(sa_size);
    for (int i = 0; i < sa_size; i++)
@@ -836,14 +760,14 @@ inline void Array<T>::operator=(const T &a)
 {
    for (int i = 0; i < size; i++)
    {
-      data[i] = a;
+      ((T*)data)[i] = a;
    }
 }
 
 template <class T>
 inline void Array<T>::Assign(const T *p)
 {
-   data.CopyFromHost(p, Size());
+   memcpy(data, p, Size()*sizeof(T));
 }
 
 

general/cuda.cpp

@@ -10,38 +10,17 @@
 // Software Foundation) version 2.1 dated February 1999.
 
 #include "cuda.hpp"
-#include "globals.hpp"
 
 namespace mfem
 {
 
-// Internal debug option, useful for tracking CUDA allocations, deallocations
-// and transfers.
-// #define MFEM_TRACK_CUDA_MEM
-
-#ifdef MFEM_USE_CUDA
-void mfem_cuda_error(cudaError_t err, const char *expr, const char *func,
-                     const char *file, int line)
-{
-   mfem::err << "\n\nCUDA error: (" << expr << ") failed with error:\n --> "
-             << cudaGetErrorString(err)
-             << "\n ... in function: " << func
-             << "\n ... in file: " << file << ':' << line << '\n';
-   mfem_error();
-}
-#endif
-
 void* CuMemAlloc(void** dptr, size_t bytes)
 {
 #ifdef MFEM_USE_CUDA
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "CuMemAlloc(): allocating " << bytes << " bytes ... "
-             << std::flush;
-#endif
-   MFEM_GPU_CHECK(cudaMalloc(dptr, bytes));
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "done: " << *dptr << std::endl;
-#endif
+   if (CUDA_SUCCESS != ::cuMemAlloc((CUdeviceptr*)dptr, bytes))
+   {
+      mfem_error("Error in CuMemAlloc");
+   }
 #endif
    return *dptr;
 }
@@ -49,14 +28,10 @@ void* CuMemAlloc(void** dptr, size_t bytes)
 void* CuMemFree(void *dptr)
 {
 #ifdef MFEM_USE_CUDA
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "CuMemFree(): deallocating memory @ " << dptr << " ... "
-             << std::flush;
-#endif
-   MFEM_GPU_CHECK(cudaFree(dptr));
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "done." << std::endl;
-#endif
+   if (CUDA_SUCCESS != ::cuMemFree((CUdeviceptr)dptr))
+   {
+      mfem_error("Error in CuMemFree");
+   }
 #endif
    return dptr;
 }
@@ -64,68 +39,70 @@ void* CuMemFree(void *dptr)
 void* CuMemcpyHtoD(void* dst, const void* src, size_t bytes)
 {
 #ifdef MFEM_USE_CUDA
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "CuMemcpyHtoD(): copying " << bytes << " bytes from "
-             << src << " to " << dst << " ... " << std::flush;
-#endif
-   MFEM_GPU_CHECK(cudaMemcpy(dst, src, bytes, cudaMemcpyHostToDevice));
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "done." << std::endl;
-#endif
+   if (CUDA_SUCCESS != ::cuMemcpyHtoD((CUdeviceptr)dst, src, bytes))
+   {
+      mfem_error("Error in CuMemcpyHtoD");
+   }
 #endif
    return dst;
 }
 
-void* CuMemcpyHtoDAsync(void* dst, const void* src, size_t bytes)
+void* CuMemcpyHtoDAsync(void* dst, const void* src, size_t bytes, void *s)
 {
 #ifdef MFEM_USE_CUDA
-   MFEM_GPU_CHECK(cudaMemcpyAsync(dst, src, bytes, cudaMemcpyHostToDevice));
+   if (CUDA_SUCCESS !=
+       ::cuMemcpyHtoDAsync((CUdeviceptr)dst, src, bytes, (CUstream)s))
+   {
+      mfem_error("Error in CuMemcpyHtoDAsync");
+   }
 #endif
    return dst;
 }
 
-void* CuMemcpyDtoD(void *dst, const void *src, size_t bytes)
+void* CuMemcpyDtoD(void* dst, void* src, size_t bytes)
 {
 #ifdef MFEM_USE_CUDA
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "CuMemcpyDtoD(): copying " << bytes << " bytes from "
-             << src << " to " << dst << " ... " << std::flush;
-#endif
-   MFEM_GPU_CHECK(cudaMemcpy(dst, src, bytes, cudaMemcpyDeviceToDevice));
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "done." << std::endl;
-#endif
+   if (CUDA_SUCCESS !=
+       ::cuMemcpyDtoD((CUdeviceptr)dst, (CUdeviceptr)src, bytes))
+   {
+      mfem_error("Error in CuMemcpyDtoD");
+   }
 #endif
    return dst;
 }
 
-void* CuMemcpyDtoDAsync(void* dst, const void *src, size_t bytes)
+void* CuMemcpyDtoDAsync(void* dst, void* src, size_t bytes, void *s)
 {
 #ifdef MFEM_USE_CUDA
-   MFEM_GPU_CHECK(cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToDevice));
+   if (CUDA_SUCCESS !=
+       ::cuMemcpyDtoDAsync((CUdeviceptr)dst, (CUdeviceptr)src,
+                           bytes, (CUstream)s))
+   {
+      mfem_error("Error in CuMemcpyDtoDAsync");
+   }
 #endif
    return dst;
 }
 
-void* CuMemcpyDtoH(void *dst, const void *src, size_t bytes)
+void* CuMemcpyDtoH(void *dst, void *src, size_t bytes)
 {
 #ifdef MFEM_USE_CUDA
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "CuMemcpyDtoH(): copying " << bytes << " bytes from "
-             << src << " to " << dst << " ... " << std::flush;
-#endif
-   MFEM_GPU_CHECK(cudaMemcpy(dst, src, bytes, cudaMemcpyDeviceToHost));
-#ifdef MFEM_TRACK_CUDA_MEM
-   mfem::out << "done." << std::endl;
-#endif
+   if (CUDA_SUCCESS != ::cuMemcpyDtoH(dst, (CUdeviceptr)src, bytes))
+   {
+      mfem_error("Error in CuMemcpyDtoH");
+   }
 #endif
    return dst;
 }
 
-void* CuMemcpyDtoHAsync(void *dst, const void *src, size_t bytes)
+void* CuMemcpyDtoHAsync(void* dst, void* src, size_t bytes, void *s)
 {
 #ifdef MFEM_USE_CUDA
-   MFEM_GPU_CHECK(cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToHost));
+   if (CUDA_SUCCESS !=
+       ::cuMemcpyDtoHAsync(dst, (CUdeviceptr)src, bytes, (CUstream)s))
+   {
+      mfem_error("Error in CuMemcpyDtoHAsync");
+   }
 #endif
    return dst;
 }