complex-jacobi minor cleanup

Merging master into feature branch.

.appveyor.yml

@@ -15,9 +15,7 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%
 
-# Install METIS, use a mirror because the original source server is not always
-# up. Original url:
-#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
+# Install METIS
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0

.gitignore

@@ -50,6 +50,7 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
+examples/ex25-gpu
 
 examples/refined.mesh
 examples/displaced.mesh

.travis.yml

@@ -11,9 +11,6 @@
 
 language: cpp
 
-os: linux
-dist: bionic
-
 stages:
   - checks
   - tests
@@ -37,7 +34,6 @@ jobs:
 
     - stage: checks
       os: linux
-      dist: xenial
       name: "code-style"
       addons:
         apt:
@@ -56,6 +52,9 @@ jobs:
           packages:
             - doxygen
             - graphviz
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
       script:
         - cd ${TRAVIS_BUILD_DIR}
         - cd tests/scripts
@@ -70,24 +69,13 @@ jobs:
             - mpich
             - libmpich-dev
       env: MPI=YES
-      before_script:
+      script:
         - cd ${TRAVIS_BUILD_DIR}
-        - mpicxx -v
         - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
         - make all -j3
         - make test-noclean
-      script:
         - cd tests/scripts
         - ./runtest gitignore
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
 
     # ========================
     #  Optional Checks/Tests
@@ -96,7 +84,6 @@ jobs:
 
     - stage: optional
       name: "branch-history"
-      if: branch != next
       # need full git history for the binary/big files check
       git:
         depth: false
@@ -125,8 +112,6 @@ jobs:
            MPI=NO
            CODECOV=NO
            MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
 
     - os: linux
       compiler: gcc
@@ -135,8 +120,6 @@ jobs:
            MPI=NO
            CODECOV=NO
            MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
 
     - os: linux
       compiler: gcc
@@ -160,7 +143,6 @@ jobs:
            MFEM_TEST_TARGET=check
            NPROCS=2
       cache:
-        ccache: true
         directories:
           - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
@@ -191,7 +173,6 @@ jobs:
            MFEM_TEST_TARGET=test
            NPROCS=2
       cache:
-        ccache: true
         directories:
           - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
@@ -223,7 +204,6 @@ jobs:
         - make -j3
         - ctest --output-on-failure
       cache:
-        ccache: true
         directories:
           - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
@@ -241,43 +221,27 @@ jobs:
     # - parallel
 
     - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
       compiler: clang
       name: "Mac: Serial + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
       env: DEBUG=YES
            MPI=NO
            CODECOV=NO
            MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
 
     - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
       compiler: clang
       name: "Mac: Serial"
-      addons:
-        homebrew:
-          packages:
-            - ccache
       env: DEBUG=NO
            MPI=NO
            CODECOV=NO
            MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
 
     - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
       compiler: clang
       name: "Mac: Parallel + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
       env: DEBUG=YES
            MPI=YES
            CODECOV=NO
@@ -285,7 +249,6 @@ jobs:
            NPROCS=4
            TMPDIR=/tmp
       cache:
-        ccache: true
         directories:
           - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
@@ -296,13 +259,9 @@ jobs:
           rm -f Lib/*.{c,o}
 
     - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
       compiler: clang
       name: "Mac: Parallel"
-      addons:
-        homebrew:
-          packages:
-            - ccache
       env: DEBUG=NO
            MPI=YES
            CODECOV=YES
@@ -310,7 +269,6 @@ jobs:
            NPROCS=4
            TMPDIR=/tmp
       cache:
-        ccache: true
         directories:
           - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
@@ -326,19 +284,14 @@ before_install:
    #      brew install open-mpi;
    #   fi
 
-   # Disable ccache while building dependencies that are cached:
-   - echo "before \$PATH = $PATH";
-     export PATH=${PATH//\/usr\/lib\/ccache:/};
-     echo "after  \$PATH = $PATH"
-
-   # On Mac OS X, build and cache OpenMPI 2.1.6:
+   # On Mac OS X, build and cache OpenMPI 2.1.1:
    - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
         if [ ! -e $HOME/local-cached/bin/mpicc ]; then
            mkdir -p $HOME/builds && cd $HOME/builds &&
-           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
-           tar jxf openmpi-2.1.6.tar.bz2 &&
+           wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2 &&
+           tar jxf openmpi-2.1.1.tar.bz2 &&
            mkdir openmpi-build && cd openmpi-build &&
-           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
+           ../openmpi-2.1.1/configure --prefix=$HOME/local-cached &&
            make -j3 all && make install;
         fi;
         PATH=$HOME/local-cached/bin:$PATH;
@@ -399,9 +352,7 @@ install:
         echo "Serial build, not using hypre";
      fi
 
-   # METIS, use a mirror because the original source server is not always up.
-   # Original url:
-   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
+   # METIS
    - if [ $MPI == "YES" ]; then
         if [ ! -e metis-4.0/libmetis.a ]; then
            wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
@@ -414,18 +365,6 @@ install:
         fi;
      fi
 
-   # Re-enable ccache on linux; enable ccache on mac os:
-   - if [ $TRAVIS_OS_NAME == "linux" ]; then
-        export PATH="/usr/lib/ccache:$PATH";
-     else
-        if [ $TRAVIS_OS_NAME == "osx" ]; then
-           export PATH="/usr/local/opt/ccache/libexec:$PATH";
-        fi;
-     fi
-
-   - printf "which \$CC  = "; which $CC;
-     printf "which \$CXX = "; which $CXX
-
 script:
    # Compiler
    - if [ $MPI == "YES" ]; then
@@ -446,9 +385,6 @@ script:
      if [ "$CODECOV" == "YES" ]; then
         CPPFLAGS="--coverage -g";
      fi;
-     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
-        CPPFLAGS+=" -pedantic -Wall -Werror";
-     fi
 
    # Configure the library
    - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG

CHANGELOG

@@ -16,12 +16,7 @@ Meshing improvements
 - The graph linear ordering library Gecko, previously an external dependency, is
   now included directly in MFEM. As a result, Mesh::GetGeckoElementOrdering is
   always available. The interface has also been improved, see for example the
-  Mesh Explorer miniapp.
-
-- Improved Gmsh reader (version 2.2), which now supports both high-order and
-  periodic meshes. Segments, triangles, quadrilaterals, and tetrahedra are
-  supported up to order 10. Wedges and hexahedra are supported up to order 9.
-  For sample periodic meshes, see the periodic*.msh files in the data directory.
+  mesh-explorer miniapp.
 
 - Added support for finite difference-based gradient and Hessian approximation
   in the TMOP mesh optimization algorithms. This improves the accuracy of the
@@ -32,8 +27,11 @@ Meshing improvements
   the user to specify different discrete functions for controlling the
   size, aspect-ratio, orientation, and skew of elements in the mesh.
 
-- Added TMOP capability for approximate tangential mesh relaxation. Added
-  support and examples for using TMOP on mixed meshes.
+- Added TMOP capability for approximate tangential mesh relaxation.
+
+- Added support for reading periodic meshes in Gmsh format (version 2.2). See
+  for example the periodic-annulus-sector and periodic-torus-sector files in
+  the data directory.
 
 - Added complete action of the TMOP Integrator to account for the spatial
   derivatives of discrete and analytic targets.
@@ -46,7 +44,7 @@ Performance improvements
     - x86 (SSE/AVX/AVX2/AVX512),
     - Power8 & Power9 (VSX),
     - BG/Q (QPX).
-  These are disabled by default, and can be enabled with MFEM_USE_SIMD=YES.
+  These are now enabled by default, and can be disabled with MFEM_USE_SIMD=NO.
   See the new file linalg/simd.hpp and the new directory linalg/simd.
 
 Improved GPU capabilities
@@ -60,10 +58,12 @@ Improved GPU capabilities
   compute a global sparse matrix. All integrators supported by element assembly
   are also supported by full assembly. See the '-fa' option in Example 9.
 
-- Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.
-
 - Added support for BlockOperator on GPU. See the updated Example 5.
 
+- Added partial assembly and GPU support for ComplexOperator,
+  [Par]ComplexGridFunction, [Par]ComplexLinearForm, and [Par]SesquilinearForm.
+  See the updated Example 22.
+
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -153,9 +153,6 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
   the Dirichlet problem for the minimal surface equation.
 
-- Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
-  preconditioning.
-
 - Added full assembly support in Example 9/9p.
 
 - Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
@@ -167,9 +164,8 @@ New and updated examples and miniapps
   mesh based on element attributes. Any newly exposed boundary elements are
   assigned attribute numbers related to the trimmed element attributes.
 
-- Added device support in Example 5/5p.
-
-- Added the option to plot a function in Mesh Explorer.
+- Added partial assembly and device support to Example 4/4p, Example 5/5p,
+  Example 22/22p, and Example 25/25p, with diagonal preconditioning.
 
 Improved testing
 ----------------

CMakeLists.txt

@@ -89,38 +89,8 @@ enable_language(CXX)
 if (MFEM_USE_CUDA)
    # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
    cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
-   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
-   if (NOT CMAKE_CUDA_HOST_COMPILER)
-      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
    enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-   set(CMAKE_CUDA_EXTENSIONS OFF)
-   set(CUDA_FLAGS "--expt-extended-lambda")
-   if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
-   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
-      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
-      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
-         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
-      else()
-         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
-      endif()
-   else()
-      set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
-   endif()
    message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
-   if (CMAKE_VERSION VERSION_LESS 3.12.0)
-      # CMake versions 3.8 and 3.9 require this to work; 3.10 and 3.11 are not
-      # tested and may not actually need this (but should be ok to keep).
-      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
-      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
-   endif()
-   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
-       "CUDA flags set for MFEM" FORCE)
-   set(CUSPARSE_FOUND TRUE)
-   set(CUSPARSE_LIBRARIES "cusparse")
 endif()
 
 if (XSDK_ENABLE_C)
@@ -326,6 +296,22 @@ if (MFEM_USE_HIOP)
   # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()
 
+# CUDA
+if (MFEM_USE_CUDA)
+   set(CMAKE_CUDA_STANDARD 11)
+   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+   set(CMAKE_CUDA_EXTENSIONS OFF)
+   set(CMAKE_CUDA_FLAGS "-arch=${CUDA_ARCH} --expt-extended-lambda"
+       CACHE STRING "CUDA flags set for MFEM" FORCE)
+   if (MFEM_USE_MPI)
+      set(CUDA_CCBIN_COMPILER ${MPI_CXX_COMPILER})
+   else()
+      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
+   endif()
+   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
+   set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CUDA_CCBIN_COMPILER})
+endif()
+
 # OCCA
 if (MFEM_USE_OCCA)
    find_package(OCCA REQUIRED)
@@ -371,8 +357,7 @@ endif()
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
     SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE)
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")

INSTALL

@@ -663,7 +663,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED > 0.6, git-hash fe5822c.
+  Versions: libCEED >= 0.6, git-hash a970f63.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -128,15 +128,7 @@ function(add_mfem_miniapp MFEM_EXE_NAME)
   if (MFEM_USE_CUDA)
     set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
       PROPERTY LANGUAGE CUDA)
-    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
-      list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
-    else()
-      set(LIST_)
-      foreach(item IN LISTS EXTRA_OPTIONS_LIST)
-        list(APPEND LIST_ "-Xcompiler=${item}")
-      endforeach()
-      set(EXTRA_OPTIONS_LIST ${LIST_})
-    endif()
+    list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
   endif()
 
   # Actually add the executable

config/defaults.cmake

@@ -50,7 +50,7 @@ option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
-option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
+option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" ON)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

config/defaults.mk

@@ -138,7 +138,7 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
-MFEM_USE_SIMD          = NO
+MFEM_USE_SIMD          = YES
 MFEM_USE_ADIOS2        = NO
 
 # Compile and link options for zlib.
@@ -341,9 +341,9 @@ GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
 GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
-# CUDA library configuration
+# CUDA library configuration (currently not needed)
 CUDA_OPT =
-CUDA_LIB = -lcusparse
+CUDA_LIB =
 
 # HIP library configuration (currently not needed)
 HIP_OPT =

data/periodic-annulus-sector.geo

@@ -1,38 +1,13 @@
 SetFactory("OpenCASCADE");
 
-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
-
-// Set the geometry order (1, 2, ..., 9)
-order = 3;
-
-// Set the element type (3 - triangles, 4 - quadrilaterals)
-type = 3;
-
-// Number of radial elements
-nrad = 2;
-
-// Number of azimuthal elements on inner arc
-nazm1 = 3;
-
-// Number of azimuthal elements on outer arc
-nazm2 = 5;
-
-// Note: Using type = 4 with nazm1 != nazm2 can lead to mixed meshes
-//       containing both triangles and quadrilaterals.
-
-// Inner and outer radii
 R1 = 1.0;
 R2 = 2.0;
 
-// Angular size of the sector
-Phi = Pi/3.0;
-
 Point(1) = {0.0, 0, 0, 1.0};
 Point(2) = {R1, 0, 0, 1.0};
 Point(3) = {R2, 0, 0, 1.0};
-Point(4) = {R1*Cos(Phi), R1*Sin(Phi), 0, 1.0};
-Point(5) = {R2*Cos(Phi), R2*Sin(Phi), 0, 1.0};
+Point(4) = {R1*Cos(Pi/3), R1*Sin(Pi/3), 0, 1.0};
+Point(5) = {R2*Cos(Pi/3), R2*Sin(Pi/3), 0, 1.0};
 Line(1) = {2, 3};
 Line(2) = {4, 5};
 Circle(3) = {2, 1, 4};
@@ -40,23 +15,13 @@ Circle(4) = {3, 1, 5};
 Curve Loop(5) = {1, 4, -2, -3};
 Plane Surface(1) = {5};
 
-Transfinite Curve{1} = nrad+1;
-Transfinite Curve{2} = nrad+1;
-Transfinite Curve{3} = nazm1+1;
-Transfinite Curve{4} = nazm2+1;
-
-If (nazm1 == nazm2)
-   Transfinite Surface{1};
-EndIf
-
-If (type == 4)
-   Recombine Surface {1};
-EndIf
+Transfinite Curve{1} = 7;
+Transfinite Curve{2} = 7;
+Transfinite Curve{3} = 4;
+Transfinite Curve{4} = 10;
 
 // Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Phi};
-EndIf
+Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Pi/3};
 
 // Tag surfaces and volumes with positive integers
 Physical Curve(1) = {3};
@@ -65,22 +30,8 @@ Physical Curve(3) = {1};
 Physical Curve(4) = {2};
 Physical Surface(1) = {1};
 
-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
-
 // Generate 2D mesh
 Mesh 2;
-SetOrder order;
 Mesh.MshFileVersion = 2.2;
 
-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-annulus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("annulus-sector-t%01g-o%01g.msh", type, order);
-EndIf
+Save "periodic-annulus-sector.msh";

data/periodic-annulus-sector.msh

@@ -2,191 +2,184 @@ $MeshFormat
 2.2 0 8
 $EndMeshFormat
 $Nodes
-136
+55
 1 1 0 0
 2 2 0 0
 3 0.5000000000000001 0.8660254037844386 0
 4 1 1.732050807568877 0
-5 1.5 0 0
-6 1.166666666666667 0 0
-7 1.333333333333333 0 0
+5 1.166666666666667 0 0
+6 1.333333333333333 0 0
+7 1.5 0 0
 8 1.666666666666667 0 0
 9 1.833333333333333 0 0
-10 0.7500000000000002 1.299038105676658 0
-11 0.5833333333333335 1.010362971081845 0
-12 0.6666666666666667 1.154700538379251 0
+10 0.5833333333333335 1.010362971081845 0
+11 0.6666666666666667 1.154700538379251 0
+12 0.7500000000000002 1.299038105676658 0
 13 0.8333333333333335 1.443375672974064 0
 14 0.9166666666666669 1.587713240271471 0
 15 0.9396926207859085 0.3420201433256683 0
 16 0.7660444431189786 0.6427876096865386 0
-17 0.993238357741943 0.1160929141252301 0
-18 0.9730448705798238 0.2306158707424401 0
-19 0.8936326403234125 0.4487991802004617 0
-20 0.8354878114129367 0.5495089780708056 0
-21 0.6862416378687343 0.7273736415730481 0
-22 0.597158591702787 0.8021231927550432 0
-23 1.956295201467611 0.4158233816355181 0
-24 1.827090915285202 0.8134732861515996 0
-25 1.618033988749896 1.175570504584944 0
-26 1.338261212717719 1.486289650954786 0
-27 1.995128100519648 0.1395129474882505 0
-28 1.980536137483141 0.278346201920131 0
-29 1.922523391876638 0.551274711633998 0
-30 1.879385241571817 0.6840402866513373 0
-31 1.765895185717855 0.9389431255717802 0
-32 1.696096192312853 1.059838528466408 0
-33 1.532088886237958 1.285575219373077 0
-34 1.438679600677305 1.389316740917992 0
-35 1.231322950651319 1.576021507213442 0
-36 1.118385806941496 1.658075145110082 0
-37 1.162276263405681 0.6710405135499813 0
-38 1.248615852873337 1.079531485311822 0
-39 1.559209616901855 0.5415673055003691 0
-40 1.478306597054007 0.8535007117539289 0
-41 0.9210953433941653 0.9653302893212266 0
-42 1.296548225291847 0.3150268220262836 0
-43 1.055002035226811 1.358510675893086 0
-44 1.704005774249187 0.2344032256041583 0
-45 0.6403651144647218 0.8991270322967013 0
-46 0.7807302289294435 0.9322286608089638 0
-47 0.864063562262777 1.07656622810637 0
-48 0.8070317811313885 1.187802166891514 0
-49 0.7236984477980553 1.043464599594108 0
-50 1.432182741763949 0.1050089406754279 0
-51 1.364365483527898 0.2100178813508558 0
-52 1.197698816861231 0.2100178813508558 0
-53 1.098849408430616 0.1050089406754279 0
-54 1.265516075097282 0.1050089406754278 0
-55 0.8177280765440409 0.7503018362314346 0
-56 0.869411709969103 0.8578160627763305 0
-57 0.7348318576552288 0.8316090412164392 0
-58 1.177596357123201 0.3240245957927452 0
-59 1.058644488954555 0.3330223695592067 0
-60 1.087610484537871 0.2205785356313179 0
-61 1.267619707955123 0.7318605796179638 0
-62 1.372963152504565 0.7926806456859463 0
-63 1.40174301566045 0.9288443029398934 0
-64 1.325179434266893 1.004187894125858 0
-65 1.219835989717452 0.9433678280578752 0
-66 1.191056126561566 0.8072041708039283 0
-67 1.296399571111008 0.8680242368719107 0
-68 1.532241943619239 0.645545107584889 0
-69 1.505274270336623 0.7495229096694089 0
-70 1.294587381237739 0.6278827775334439 0
-71 1.426898499069797 0.5847250415169065 0
-72 1.399930825787181 0.6887028436014264 0
-73 1.139442349713613 1.041464419981624 0
-74 1.030268846553889 1.003397354651425 0
-75 1.001488983398004 0.8672336973974781 0
-76 1.081882623401843 0.7691371054737297 0
-77 1.110662486557728 0.9053007627276766 0
-78 1.207033584034403 0.5523692830420821 0
-79 1.251790904663125 0.4336980525341829 0
-80 1.384102022495183 0.3905403165176455 0
-81 1.471655819698519 0.4660538110090073 0
-82 1.339344701866461 0.5092115470255447 0
-83 1.73779714915742 0.7228379592678562 0
-84 1.648503383029637 0.6322026323841127 0
-85 1.691571478423774 0.4996526642120855 0
-86 1.823933339945692 0.4577380229238018 0
-87 1.787039416783436 0.5922941012619014 0
-88 1.308379426102925 1.350703595740465 0
-89 1.278497639488131 1.215117540526144 0
-90 1.371755231498856 1.111544491736196 0
-91 1.494894610124376 1.14355749816057 0
-92 1.4064614465962 1.25147448186763 0
-93 1.013887168325833 0.451693600067106 0
-94 1.088081715865757 0.5613670568085436 0
-95 1.03019898997678 0.6616228789288338 0
-96 0.8981217165478794 0.6522052443076862 0
-97 0.9637989050473432 0.5564495572737495 0
-98 1.432367408277627 0.2881522898855752 0
-99 1.568186591263407 0.2612777577448668 0
-100 1.655740388466743 0.3367912522362286 0
-101 1.607475002684299 0.4391792788682989 0
-102 1.519921205480963 0.3636657843769371 0
-103 1.184077913657828 1.17252454883891 0
-104 1.119539974442319 1.265517612365998 0
-105 1.010366471282595 1.2274505470358 0
-106 0.9657309073383804 1.096390418178513 0
-107 1.074904410498104 1.134457483508712 0
-108 1.901335258083062 0.07813440853471942 0
-109 1.802670516166124 0.1562688170694388 0
-110 1.636003849499458 0.156268817069439 0
-111 1.568001924749729 0.07813440853471942 0
-112 1.734668591416396 0.07813440853471944 0
-113 0.8516673450756037 1.318862295748801 0
-114 0.9533346901512071 1.338686485820944 0
-115 1.03666802348454 1.48302405311835 0
-116 1.01833401174227 1.607537430343614 0
-117 0.9350006784089369 1.463199863046207 0
-118 1.710829475874804 0.8268157613523761 0
-119 1.594568036464405 0.8401582365531526 0
-120 1.621535709747021 0.7361804344686326 0
-121 1.52488239428597 0.9608573093642675 0
-122 1.571458191517933 1.068213906974606 0
-123 1.448318812892413 1.036200900550232 0
-124 0.908699126206992 1.207626356963657 0
-125 1.500184666513678 0.1831433492101474 0
-126 1.646765991694905 0.9507607885973723 0
-127 0.9498053499729417 0.7597194708525821 0
-128 1.132839036494479 0.4426958263006443 0
-129 1.149421761057113 1.40110366758032 0
-130 1.243841486887416 1.443696659267553 0
-131 1.134903597606542 1.530869109405537 0
-132 1.872198725728137 0.3553499962917315 0
-133 1.788102249988661 0.2948766109479449 0
-134 1.893223337417325 0.2174207916708467 0
-135 1.213959700272622 1.308110604053232 0
-136 1.739836864206217 0.3972646375800152 0
+17 1.986476715483886 0.2321858282504602 0
+18 1.946089741159648 0.4612317414848793 0
+19 1.879385241571817 0.6840402866513365 0
+20 1.787265280646825 0.8975983604009234 0
+21 1.670975622825874 1.09901795614161 0
+22 1.532088886237958 1.285575219373077 0
+23 1.372483275737469 1.454747283146095 0
+24 1.194317183405575 1.604246385510085 0
+25 1.425989114816062 0.1915326920916892 0
+26 0.8788667344146573 1.13917645290495 0
+27 1.630372059110754 0.7154531062316609 0
+28 1.436395769298814 1.053728612482506 0
+29 1.081023776188756 0.6241293681829633 0
+30 1.168737372335971 1.428012728596308 0
+31 1.821063986059922 0.298149890497067 0
+32 1.234707097211386 0.3469796339295647 0
+33 1.377747393186519 0.6200150626754309 0
+34 1.457047681210906 0.3890895843559762 0
+35 0.917846726184522 0.8957978954532204 0
+36 1.218335619030348 0.9017812086952638 0
+37 1.066623110765233 1.061857005744772 0
+38 1.587029716281926 0.1355955181472859 0
+39 1.744445799211916 0.1441515753740107 0
+40 1.25 0.1443375672974065 0
+41 1.453660070628011 0.8435769396609902 0
+42 1.741367044061892 0.499612708014486 0
+43 1.30550638526547 1.257610469847477 0
+44 1.118213276932792 0.1666674689105279 0
+45 0.9109440214958271 1.306610291787315 0
+46 0.9970618258753989 1.438658589955562 0
+47 0.7499999999999998 1.010362971081845 0
+48 0.7034449005273667 0.8850673702175776 0
+49 1.605449512513618 0.9269067082200894 0
+50 1.561654019115059 0.5298592532912715 0
+51 1.229782222487711 1.096820457143683 0
+52 1.617066998712459 0.3090202662210922 0
+53 1.079645953234324 1.246963713711438 0
+54 1.877063966817811 0.1348974588243076 0
+55 1.055356609656722 1.558136350380461 0
 $EndNodes
 $Elements
-38
-1 26 2 3 1 1 5 6 7
-2 26 2 3 1 5 2 8 9
-3 26 2 4 2 3 10 11 12
-4 26 2 4 2 10 4 13 14
-5 26 2 1 3 1 15 17 18
-6 26 2 1 3 15 16 19 20
-7 26 2 1 3 16 3 21 22
-8 26 2 2 4 2 23 27 28
-9 26 2 2 4 23 24 29 30
-10 26 2 2 4 24 25 31 32
-11 26 2 2 4 25 26 33 34
-12 26 2 2 4 26 4 35 36
-13 21 2 1 1 3 41 10 45 46 47 48 12 11 49
-14 21 2 1 1 5 42 1 50 51 52 53 6 7 54
-15 21 2 1 1 16 41 3 55 56 46 45 22 21 57
-16 21 2 1 1 1 42 15 53 52 58 59 18 17 60
-17 21 2 1 1 37 40 38 61 62 63 64 65 66 67
-18 21 2 1 1 39 40 37 68 69 62 61 70 71 72
-19 21 2 1 1 38 41 37 73 74 75 76 66 65 77
-20 21 2 1 1 37 42 39 78 79 80 81 71 70 82
-21 21 2 1 1 24 39 23 83 84 85 86 29 30 87
-22 21 2 1 1 26 38 25 88 89 90 91 33 34 92
-23 21 2 1 1 15 37 16 93 94 95 96 20 19 97
-24 21 2 1 1 42 44 39 98 99 100 101 81 80 102
-25 21 2 1 1 38 43 41 103 104 105 106 74 73 107
-26 21 2 1 1 2 44 5 108 109 110 111 8 9 112
-27 21 2 1 1 10 43 4 113 114 115 116 14 13 117
-28 21 2 1 1 24 40 39 118 119 69 68 84 83 120
-29 21 2 1 1 38 40 25 64 63 121 122 91 90 123
-30 21 2 1 1 41 43 10 106 105 114 113 48 47 124
-31 21 2 1 1 5 44 42 111 110 99 98 51 50 125
-32 21 2 1 1 25 40 24 122 121 119 118 31 32 126
-33 21 2 1 1 37 41 16 76 75 56 55 96 95 127
-34 21 2 1 1 15 42 37 59 58 79 78 94 93 128
-35 21 2 1 1 4 43 26 116 115 129 130 35 36 131
-36 21 2 1 1 23 44 2 132 133 109 108 27 28 134
-37 21 2 1 1 26 43 38 130 129 104 103 89 88 135
-38 21 2 1 1 39 44 23 101 100 133 132 86 85 136
+108
+1 1 2 3 1 1 5
+2 1 2 3 1 5 6
+3 1 2 3 1 6 7
+4 1 2 3 1 7 8
+5 1 2 3 1 8 9
+6 1 2 3 1 9 2
+7 1 2 4 2 3 10
+8 1 2 4 2 10 11
+9 1 2 4 2 11 12
+10 1 2 4 2 12 13
+11 1 2 4 2 13 14
+12 1 2 4 2 14 4
+13 1 2 1 3 1 15
+14 1 2 1 3 15 16
+15 1 2 1 3 16 3
+16 1 2 2 4 2 17
+17 1 2 2 4 17 18
+18 1 2 2 4 18 19
+19 1 2 2 4 19 20
+20 1 2 2 4 20 21
+21 1 2 2 4 21 22
+22 1 2 2 4 22 23
+23 1 2 2 4 23 24
+24 1 2 2 4 24 4
+25 2 2 1 1 32 40 25
+26 2 2 1 1 25 34 32
+27 2 2 1 1 33 41 36
+28 2 2 1 1 38 52 25
+29 2 2 1 1 33 36 29
+30 2 2 1 1 26 47 35
+31 2 2 1 1 35 37 26
+32 2 2 1 1 25 52 34
+33 2 2 1 1 32 44 40
+34 2 2 1 1 15 32 29
+35 2 2 1 1 15 29 16
+36 2 2 1 1 36 41 28
+37 2 2 1 1 32 33 29
+38 2 2 1 1 50 52 42
+39 2 2 1 1 32 34 33
+40 2 2 1 1 42 52 31
+41 2 2 1 1 43 53 51
+42 2 2 1 1 27 41 33
+43 2 2 1 1 26 53 45
+44 2 2 1 1 18 31 17
+45 2 2 1 1 29 35 16
+46 2 2 1 1 29 36 35
+47 2 2 1 1 24 30 23
+48 2 2 1 1 30 53 43
+49 2 2 1 1 17 54 2
+50 2 2 1 1 4 55 24
+51 2 2 1 1 28 51 36
+52 2 2 1 1 47 48 35
+53 2 2 1 1 36 37 35
+54 2 2 1 1 37 53 26
+55 2 2 1 1 22 28 21
+56 2 2 1 1 20 27 19
+57 2 2 1 1 33 50 27
+58 2 2 1 1 15 44 32
+59 2 2 1 1 18 42 31
+60 2 2 1 1 30 43 23
+61 2 2 1 1 35 48 16
+62 2 2 1 1 31 54 17
+63 2 2 1 1 9 39 8
+64 2 2 1 1 8 38 7
+65 2 2 1 1 7 25 6
+66 2 2 1 1 22 43 28
+67 2 2 1 1 23 43 22
+68 2 2 1 1 39 54 31
+69 2 2 1 1 19 42 18
+70 2 2 1 1 24 55 30
+71 2 2 1 1 27 42 19
+72 2 2 1 1 13 46 14
+73 2 2 1 1 51 53 37
+74 2 2 1 1 39 52 38
+75 2 2 1 1 6 40 5
+76 2 2 1 1 34 52 50
+77 2 2 1 1 12 45 13
+78 2 2 1 1 30 55 46
+79 2 2 1 1 10 47 11
+80 2 2 1 1 8 39 38
+81 2 2 1 1 28 49 21
+82 2 2 1 1 7 38 25
+83 2 2 1 1 41 49 28
+84 2 2 1 1 20 49 27
+85 2 2 1 1 11 26 12
+86 2 2 1 1 27 49 41
+87 2 2 1 1 31 52 39
+88 2 2 1 1 25 40 6
+89 2 2 1 1 2 54 9
+90 2 2 1 1 14 55 4
+91 2 2 1 1 45 53 46
+92 2 2 1 1 45 46 13
+93 2 2 1 1 5 44 1
+94 2 2 1 1 21 49 20
+95 2 2 1 1 46 53 30
+96 2 2 1 1 3 48 10
+97 2 2 1 1 34 50 33
+98 2 2 1 1 36 51 37
+99 2 2 1 1 26 45 12
+100 2 2 1 1 11 47 26
+101 2 2 1 1 27 50 42
+102 2 2 1 1 40 44 5
+103 2 2 1 1 43 51 28
+104 2 2 1 1 10 48 47
+105 2 2 1 1 9 54 39
+106 2 2 1 1 46 55 14
+107 2 2 1 1 1 44 15
+108 2 2 1 1 16 48 3
 $EndElements
 $Periodic
 1
 1 1 2
 Affine 0.5000000000000001 0.8660254037844386 0 0 -0.8660254037844386 0.5000000000000001 0 0 0 0 1 0 0 0 0 1
-3
+7
+9 14
+6 11
+8 13
 5 10
-1 3
+7 12
 2 4
+1 3
 $EndPeriodic

data/periodic-torus-sector.geo

@@ -1,141 +1,25 @@
-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
+SetFactory("OpenCASCADE");
 
-// Set the geometry order (1, 2, ..., 10 for tetrahedra or 9 for other types)
-order = 3;
+R = 1.5;
+r = 0.5;
 
-// Set the element type (4 - tetrahedra, 6 - wedges, 8 - hexahedra)
-type = 8;
+Torus(1) = {0,0,0, R, r, Pi/3};
 
-// Minor and major radii
-R1 = 1.0;
-R2 = 2.0;
+pts() = PointsOf{ Volume{1}; };
 
-// Side length of interior square
-A1 = 0.8;
-
-// Angular size of the sector
-Phi = Pi/3.0;
-
-// Number of azimuthal elements
-nazm = 3;
-
-// Number of elements around a quarter of the circle
-narc = 2;
-
-// Number of elements between surface and interior square
-nshl = 1;
-
-lc = 0.5;
-a1 = A1 / Sqrt(2.0);
-
-Point(1) = {R2+R1, 0, 0, lc};
-Point(2) = {R2, 0, R1, lc};
-Point(3) = {R2-R1, 0, 0, lc};
-Point(4) = {R2, 0, -R1, lc};
-Point(5) = {R2, 0, 0, lc};
-Point(6) = {R2+a1, 0, 0, lc};
-Point(7) = {R2, 0, a1, lc};
-Point(8) = {R2-a1, 0, 0, lc};
-Point(9) = {R2, 0, -a1, lc};
-
-Circle(1) = {1,5,2};
-Circle(2) = {2,5,3};
-Circle(3) = {3,5,4};
-Circle(4) = {4,5,1};
-
-Line(5) = {6,1};
-Line(6) = {7,2};
-Line(7) = {8,3};
-Line(8) = {9,4};
-
-Line(9) = {6, 7};
-Line(10) = {7, 8};
-Line(11) = {8, 9};
-Line(12) = {9, 6};
-
-Line Loop(101) = {1, -6, -9, 5};
-Line Loop(102) = {2, -7, -10, 6};
-Line Loop(103) = {3, -8, -11, 7};
-Line Loop(104) = {4, -5, -12, 8};
-Line Loop(105) = {9, 10, 11, 12};
-
-Plane Surface(201) = {101};
-Plane Surface(202) = {102};
-Plane Surface(203) = {103};
-Plane Surface(204) = {104};
-Plane Surface(205) = {105};
-
-Transfinite Curve{1} = narc+1;
-Transfinite Curve{2} = narc+1;
-Transfinite Curve{3} = narc+1;
-Transfinite Curve{4} = narc+1;
-
-Transfinite Curve{5} = nshl+1;
-Transfinite Curve{6} = nshl+1;
-Transfinite Curve{7} = nshl+1;
-Transfinite Curve{8} = nshl+1;
-
-Transfinite Curve{9} = narc+1;
-Transfinite Curve{10} = narc+1;
-Transfinite Curve{11} = narc+1;
-Transfinite Curve{12} = narc+1;
-
-If (type == 8)
-   Recombine Surface {201};
-   Recombine Surface {202};
-   Recombine Surface {203};
-   Recombine Surface {204};
-   Recombine Surface {205};
-
-   Transfinite Surface {201} = {1,2,7,6};
-   Transfinite Surface {202} = {2,3,8,7};
-   Transfinite Surface {203} = {3,4,9,8};
-   Transfinite Surface {204} = {4,1,6,9};
-   Transfinite Surface {205} = {6,7,8,9};
-EndIf
-
-If (type == 4)
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm};
-}
-Else
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm}; Recombine;
-}
-EndIf
+Characteristic Length{ pts() } = 0.25;
 
 // Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Surface{227} = {201} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{249} = {202} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{271} = {203} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{293} = {204} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{315} = {205} Rotate{{0,0,1}, {0,0,0}, Phi};
-EndIf
+Periodic Surface{3} = {2} Rotate{{0,0,1}, {0,0,0}, Pi/3};
 
 // Tag surfaces and volumes with positive integers
-Physical Surface(1) = {201,202,203,204,205};
-Physical Surface(2) = {227,249,271,293,315};
-Physical Surface(3) = {214,236,258,280};
-Physical Volume(1) = {1,2,3,4,5};
-
-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
+Physical Surface(1) = {1};
+Physical Surface(2) = {2};
+Physical Surface(3) = {3};
+Physical Volume(1) = {1};
 
 // Generate 3D mesh
 Mesh 3;
-SetOrder order;
+
 Mesh.MshFileVersion = 2.2;
-
-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-torus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("torus-sector-t%01g-o%01g.msh", type, order);
-EndIf
+Save "periodic-torus-sector.msh";

data/square-mixed.mesh

@@ -1,118 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-20
-1 3 0 1 6 5
-1 3 1 2 7 6
-1 3 2 3 8 7
-1 3 3 4 9 8
-1 3 5 6 11 10
-1 2 6 7 11
-1 2 7 12 11
-1 2 7 8 13
-1 2 7 13 12
-1 3 8 9 14 13
-1 3 10 11 16 15
-1 2 11 12 17
-1 2 11 17 16
-1 2 12 13 17
-1 2 13 18 17
-1 3 13 14 19 18
-1 3 15 16 21 20
-1 3 16 17 22 21
-1 3 17 18 23 22
-1 3 18 19 24 23
-
-boundary
-16
-2 1 0 1
-2 1 1 2
-2 1 2 3
-2 1 3 4
-2 1 21 20
-2 1 22 21
-2 1 23 22
-2 1 24 23
-1 1 5 0
-1 1 10 5
-1 1 15 10
-1 1 20 15
-1 1 4 9
-1 1 9 14
-1 1 14 19
-1 1 19 24
-
-vertices
-25
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P1
-VDim: 2
-Ordering: 0
-
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0
-0
-0
-0
-0.25
-0.25
-0.25
-0.25
-0.25
-0.5
-0.5
-0.5
-0.5
-0.5
-0.75
-0.75
-0.75
-0.75
-0.75
-1
-1
-1
-1
-1

doc/CodeDocumentation.dox

@@ -144,7 +144,8 @@ namespace mfem {
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
  * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
- * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
+
+* - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
  * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
  * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes

doc/makefile

@@ -19,7 +19,7 @@ html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
 	@doxygen $(DOXYGEN_CONF)
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
-	@cat warnings.log 1>&2
+	@cat warnings.log
 	@# Generate the log of undocumented methods
 	@( cat $(DOXYGEN_CONF) ; echo "GENERATE_HTML=NO" ; echo "EXTRACT_ALL=NO" ; echo "WARN_LOGFILE=undoc.log" ; echo "QUIET=YES" ) | doxygen - &> /dev/null
 

examples/ex1.cpp

@@ -175,8 +175,7 @@ int main(int argc, char *argv[])
    //    domain integrator.
    BilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   a.AddDomainIntegrator(new MassIntegrator(one));
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: eliminating boundary
@@ -185,40 +184,19 @@ int main(int argc, char *argv[])
    if (static_cond) { a.EnableStaticCondensation(); }
    a.Assemble();
 
-   OperatorPtr A, As;
+   OperatorPtr A;
    Vector B, X;
-   Array<int> empty_list;
-   a.FormSystemMatrix(empty_list, As);
-   //a.FormLinearSystem(empty_list, x, b, A, X, B);
-   //a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   //cout << "Size of linear system: " << A->Height() << endl;
+   cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
    if (!pa)
    {
 #ifndef MFEM_USE_SUITESPARSE
       // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      //GSSmoother M((SparseMatrix&)(*A));
-
-      //SparseMatrix &Asp = *As.As<SparseMatrix>();
-      SparseMatrix &Asp = a.SpMat();
-
-      Asp.Finalize();
-      Asp.SortColumnIndices();
-
-      Vector tmpx(B.Size());
-      Vector tmpy(B.Size());
-      tmpx = 1.0;
-      tmpy = 0.0;
-
-      //As.As<SparseMatrix>()->Mult(tmpx, tmpy);
-      Asp.Mult(tmpx, tmpy);
-
-      //IncompleteCholesky M(*As.As<SparseMatrix>());
-      IncompleteCholesky M(Asp);
-      //ILUcusparse M(*A.As<SparseMatrix>());
-      PCG(*As, M, B, X, 1, 200, 1e-12, 0.0);
+      GSSmoother M((SparseMatrix&)(*A));
+      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
 #else
       // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
       UMFPackSolver umf_solver;

examples/ex1p.cpp

@@ -122,7 +122,7 @@ int main(int argc, char *argv[])
    {
       int ref_levels =
          (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels-1; l++)
+      for (int l = 0; l < ref_levels; l++)
       {
          mesh.UniformRefinement();
       }
@@ -134,7 +134,7 @@ int main(int argc, char *argv[])
    ParMesh pmesh(MPI_COMM_WORLD, mesh);
    mesh.Clear();
    {
-      int par_ref_levels = 1;
+      int par_ref_levels = 2;
       for (int l = 0; l < par_ref_levels; l++)
       {
          pmesh.UniformRefinement();
@@ -216,13 +216,6 @@ int main(int argc, char *argv[])
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   SparseMatrix Asp;
-   A.As<HypreParMatrix>()->GetDiag(Asp);
-   Vector diag;
-
-   StopWatch sw;
-   sw.Start();
-
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
    //     * With partial assembly, use Jacobi smoothing, for now.
@@ -236,14 +229,7 @@ int main(int argc, char *argv[])
    }
    else
    {
-      //prec = new HypreBoomerAMG;
-      Asp.Finalize();
-      Asp.SortColumnIndices();
-
-      Asp.GetDiag(diag);
-      prec = new OperatorJacobiSmoother(diag, ess_tdof_list);
-      //prec = new IncompleteCholesky(Asp);
-      //prec = new ILUcusparse(Asp);
+      prec = new HypreBoomerAMG;
    }
    CGSolver cg(MPI_COMM_WORLD);
    cg.SetRelTol(1e-12);
@@ -254,9 +240,6 @@ int main(int argc, char *argv[])
    cg.Mult(B, X);
    delete prec;
 
-   sw.Stop();
-   cout << "Step 13 solve time " << sw.RealTime() << endl;
-
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a.RecoverFEMSolution(X, b, x);

examples/ex22.cpp

@@ -13,10 +13,10 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// With partial assembly:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa
+// Device sample runs:
+//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
+//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
+//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -82,6 +82,7 @@ int main(int argc, char *argv[])
    bool herm_conv = true;
    bool exact_sol = true;
    bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -114,6 +115,8 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -143,13 +146,18 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
    //    with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Refine the mesh to increase resolution. In this example we do
+   // 4. Refine the mesh to increase resolution. In this example we do
    //    'ref_levels' of uniform refinement where the user specifies
    //    the number of levels with the '-r' option.
    for (int l = 0; l < ref_levels; l++)
@@ -157,7 +165,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 4. Define a finite element space on the mesh. Here we use continuous
+   // 5. Define a finite element space on the mesh. Here we use continuous
    //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
    //    order.
    if (dim == 1 && prob != 0 )
@@ -179,7 +187,7 @@ int main(int argc, char *argv[])
    cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
         << endl;
 
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined based on the type
    //    of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -191,12 +199,12 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system.
    ComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 7. Define the solution vector u as a complex finite element grid function
+   // 8. Define the solution vector u as a complex finite element grid function
    //    corresponding to fespace. Initialize u with initial guess of 1+0i or
    //    the exact solution if it is known.
    ComplexGridFunction u(fespace);
@@ -218,7 +226,6 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
-   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -271,7 +278,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 8. Set up the sesquilinear form a(.,.) on the finite element space
+   // 9. Set up the sesquilinear form a(.,.) on the finite element space
    //    corresponding to the damped harmonic oscillator operator of the
    //    appropriate type:
    //
@@ -314,7 +321,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 8a. Set up the bilinear form for the preconditioner corresponding to the
+   // 9a. Set up the bilinear form for the preconditioner corresponding to the
    //     appropriate operator
    //
    //      0) A scalar H1 field
@@ -349,9 +356,9 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 9. Assemble the form and the corresponding linear system, applying any
-   //    necessary transformations such as: assembly, eliminating boundary
-   //    conditions, conforming constraints for non-conforming AMR, etc.
+   // 10. Assemble the form and the corresponding linear system, applying any
+   //     necessary transformations such as: assembly, eliminating boundary
+   //     conditions, conforming constraints for non-conforming AMR, etc.
    a->Assemble();
    pcOp->Assemble();
 
@@ -362,7 +369,7 @@ int main(int argc, char *argv[])
 
    cout << "Size of linear system: " << A->Width() << endl << endl;
 
-   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
+   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
    //     preconditioner based on the appropriate sparse smoother.
    {
       Array<int> blockOffsets;
@@ -419,9 +426,11 @@ int main(int argc, char *argv[])
       gmres.Mult(B, U);
    }
 
-   // 11. Recover the solution as a finite element grid function and compute the
+   // 12. Recover the solution as a finite element grid function and compute the
    //     errors if the exact solution is known.
    a->RecoverFEMSolution(U, b, u);
+   u.real().SyncMemory(u);
+   u.imag().SyncMemory(u);
 
    if (exact_sol)
    {
@@ -451,7 +460,7 @@ int main(int argc, char *argv[])
       cout << endl;
    }
 
-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -m mesh -g sol".
    {
       ofstream mesh_ofs("refined.mesh");
@@ -466,7 +475,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -525,7 +534,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 14. Free the used memory.
+   // 15. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex22p.cpp

@@ -13,10 +13,10 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// With partial assembly:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa
+// Device sample runs:
+//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
+//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
+//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -46,7 +46,6 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.
 
-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -90,6 +89,7 @@ int main(int argc, char *argv[])
    bool herm_conv = true;
    bool exact_sol = true;
    bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -124,6 +124,8 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -160,19 +162,24 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the serial mesh on all processors to increase the resolution.
+   // 5. Refine the serial mesh on all processors to increase the resolution.
    for (int l = 0; l < ser_ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -182,7 +189,7 @@ int main(int argc, char *argv[])
       pmesh->UniformRefinement();
    }
 
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
    //    the specified order.
    if (dim == 1 && prob != 0 )
@@ -210,7 +217,7 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 7. Determine the list of true (i.e. parallel conforming) essential
+   // 8. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined
    //    based on the type of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -222,14 +229,14 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 8. Set up the parallel linear form b(.) which corresponds to the
+   // 9. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system.
    ParComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 9. Define the solution vector u as a parallel complex finite element grid
-   //    function corresponding to fespace. Initialize u with initial guess of
-   //    1+0i or the exact solution if it is known.
+   // 10. Define the solution vector u as a parallel complex finite element grid
+   //     function corresponding to fespace. Initialize u with initial guess of
+   //     1+0i or the exact solution if it is known.
    ParComplexGridFunction u(fespace);
    ParComplexGridFunction * u_exact = NULL;
    if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -249,7 +256,6 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
-   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -304,7 +310,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
    //     space corresponding to the damped harmonic oscillator operator of the
    //     appropriate type:
    //
@@ -347,7 +353,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 10a. Set up the parallel bilinear form for the preconditioner
+   // 11a. Set up the parallel bilinear form for the preconditioner
    //      corresponding to the appropriate operator
    //
    //      0) A scalar H1 field
@@ -381,7 +387,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 11. Assemble the parallel bilinear form and the corresponding linear
+   // 12. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, etc.
@@ -399,7 +405,7 @@ int main(int argc, char *argv[])
            << 2 * fespace->GlobalTrueVSize() << endl << endl;
    }
 
-   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
    //     diagonal preconditioner based on the appropriate multigrid
    //     preconditioner from hypre.
    {
@@ -460,9 +466,11 @@ int main(int argc, char *argv[])
       fgmres.SetPrintLevel(1);
       fgmres.Mult(B, U);
    }
-   // 13. Recover the parallel grid function corresponding to U. This is the
+   // 14. Recover the parallel grid function corresponding to U. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(U, b, u);
+   u.real().SyncMemory(u);
+   u.imag().SyncMemory(u);
 
    if (exact_sol)
    {
@@ -495,7 +503,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 14. Save the refined mesh and the solution in parallel. This output can be
+   // 15. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -515,7 +523,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 15. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -580,7 +588,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 16. Free the used memory.
+   // 17. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex25.cpp

@@ -10,6 +10,10 @@
 //               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
 //               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh
 //
+// Device sample runs:
+//               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh -pa -d cuda
+//               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh -pa -d cuda
+//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -82,24 +86,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLMatrixCoefficient : public MatrixCoefficient
+class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
-   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                              Vector &),
+                            CartesianPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+   virtual void Eval(Vector &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(height, width);
+      K.SetSize(vdim);
       (*Function)(transip, pml, K);
    }
 };
@@ -116,13 +120,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -153,6 +157,8 @@ int main(int argc, char *argv[])
    double freq = 5.0;
    bool herm_conv = true;
    bool visualization = 1;
+   bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -174,12 +180,21 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
 
    if (iprob > 4) { iprob = 4; }
    prob = (prob_type)iprob;
 
-   // 2. Setup the mesh
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Setup the mesh
    if (!mesh_file)
    {
       exact_known = true;
@@ -220,7 +235,7 @@ int main(int argc, char *argv[])
    // Setup PML length
    Array2D<double> length(dim, 2); length = 0.0;
 
-   // 3. Setup the Cartesian PML region.
+   // 4. Setup the Cartesian PML region.
    switch (prob)
    {
       case disc:
@@ -246,19 +261,19 @@ int main(int argc, char *argv[])
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
-   // 4. Refine the mesh to increase the resolution.
+   // 5. Refine the mesh to increase the resolution.
    for (int l = 0; l < ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 5. Reorient mesh in case of a tet mesh
+   // 6. Reorient mesh in case of a tet mesh
    mesh->ReorientTetMesh();
 
    // Set element attributes in order to distinguish elements in the PML region
    pml->SetAttributes(mesh);
 
-   // 6. Define a finite element space on the mesh. Here we use the Nedelec
+   // 7. Define a finite element space on the mesh. Here we use the Nedelec
    //    finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
@@ -266,7 +281,7 @@ int main(int argc, char *argv[])
 
    cout << "Number of finite element unknowns: " << size << endl;
 
-   // 7. Determine the list of true essential boundary dofs. In this example,
+   // 8. Determine the list of true essential boundary dofs. In this example,
    //    the boundary conditions are defined based on the specific mesh and the
    //    problem type.
    Array<int> ess_tdof_list;
@@ -308,12 +323,12 @@ int main(int argc, char *argv[])
    }
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 8. Setup Complex Operator convention
+   // 9. Setup Complex Operator convention
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
+   // 10. Set up the linear form b(.) which corresponds to the right-hand side of
+   //     the FEM linear system.
    VectorFunctionCoefficient f(dim, source);
    ComplexLinearForm b(fespace, conv);
    if (prob == load_src)
@@ -323,7 +338,7 @@ int main(int argc, char *argv[])
    b.Vector::operator=(0.0);
    b.Assemble();
 
-   // 10. Define the solution vector x as a complex finite element grid function
+   // 11. Define the solution vector x as a complex finite element grid function
    //     corresponding to fespace.
    ComplexGridFunction x(fespace);
    x = 0.0;
@@ -331,7 +346,7 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
    x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
 
-   // 11. Set up the sesquilinear form a(.,.)
+   // 12. Set up the sesquilinear form a(.,.)
    //
    //     In Comp
    //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -365,19 +380,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
-   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
+   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
-   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
+   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -385,26 +400,30 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                          new VectorFEMassIntegrator(restr_c2_Im));
 
-   // 12. Assemble the bilinear form and the corresponding linear system,
+   // 13. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: assembly, eliminating
    //     boundary conditions, applying conforming constraints for
    //     non-conforming AMR, etc.
+#ifndef MFEM_USE_SUITESPARSE
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+#endif
    a.Assemble(0);
 
    OperatorPtr A;
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   // 13. Solve using a direct or an iterative solver
+   // 14. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
    {
+      if (pa) { cout << "PA not available with MFEM_USE_SUITESPARSE" << endl; }
       ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
       csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
       csolver.SetPrintLevel(1);
       csolver.Mult(B, X);
    }
 #else
-   // 13a. Set up the Bilinear form a(.,.) for the preconditioner
+   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
    //
    //    In Comp
    //    Domain:   1/mu (Curl E, Curl F) + omega^2 * epsilon (E,F)
@@ -419,50 +438,69 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
-      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
+      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
 
+      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
       prec.Assemble();
 
-      OperatorPtr PCOpAh;
-      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-      // 13b. Define and apply a GMRES solver for AU=B with a block diagonal
-      //      preconditioner based on the Gauss-Seidel sparse smoother.
+      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
+      //      preconditioner based on the Gauss-Seidel or Jacobi sparse smoother.
       Array<int> offsets(3);
       offsets[0] = 0;
       offsets[1] = fespace->GetTrueVSize();
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      GSSmoother gs00(*PCOpAh.As<SparseMatrix>());
-      BlockDiagonalPreconditioner BlockGS(offsets);
-      ScaledOperator gs11(&gs00,
-                          (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
-      BlockGS.SetDiagonalBlock(0,&gs00);
-      BlockGS.SetDiagonalBlock(1,&gs11);
+      Operator *pc_r = nullptr;
+      Operator *pc_i = nullptr;
+      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      if (pa)
+      {
+         // Jacobi Smoother
+         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
+         ScaledOperator *d11 = new ScaledOperator(d00, s);
+         pc_r = d00;
+         pc_i = d11;
+      }
+      else
+      {
+         OperatorPtr PCOpAh;
+         prec.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
+         // Gauss-Seidel Smoother
+         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
+         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
+         pc_r = gs00;
+         pc_i = gs11;
+      }
+
+      BlockDiagonalPreconditioner BlockDP(offsets);
+      BlockDP.SetDiagonalBlock(0, pc_r);
+      BlockDP.SetDiagonalBlock(1, pc_i);
 
       GMRESSolver gmres;
       gmres.SetPrintLevel(1);
       gmres.SetKDim(200);
-      gmres.SetMaxIter(2000);
+      gmres.SetMaxIter(pa ? 5000 : 2000);
       gmres.SetRelTol(1e-5);
       gmres.SetAbsTol(0.0);
       gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockGS);
+      gmres.SetPreconditioner(BlockDP);
       gmres.Mult(B, X);
    }
 #endif
 
-   // 14. Recover the solution as a finite element grid function and compute the
+   // 15. Recover the solution as a finite element grid function and compute the
    //     errors if the exact solution is known.
    a.RecoverFEMSolution(X, b, x);
 
@@ -499,7 +537,7 @@ int main(int argc, char *argv[])
            << sqrt(L2Error_Re*L2Error_Re + L2Error_Im*L2Error_Im) << "\n\n";
    }
 
-   // 15. Save the refined mesh and the solution. This output can be viewed
+   // 16. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -m mesh -g sol".
    {
       ofstream mesh_ofs("ex25.mesh");
@@ -514,7 +552,7 @@ int main(int argc, char *argv[])
       x.imag().Save(sol_i_ofs);
    }
 
-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       // Define visualization keys for GLVis (see GLVis documentation)
@@ -565,7 +603,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 17. Free the used memory.
+   // 18. Free the used memory.
    delete pml;
    delete fespace;
    delete fec;
@@ -763,7 +801,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -774,14 +812,13 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -792,14 +829,13 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -810,14 +846,13 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -831,19 +866,18 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      M = (1.0 / det).real();
+      D = (1.0 / det).real();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -856,19 +890,18 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = (1.0 / det).imag();
+      D = (1.0 / det).imag();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -881,14 +914,13 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = abs(1.0 / det);
+      D = abs(1.0 / det);
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

examples/ex25p.cpp

@@ -10,6 +10,10 @@
 //               mpirun -np 4 ex25p -o 2 -f 8.0 -rs 2 -rp 2 -prob 4 -m ../data/inline-quad.mesh
 //               mpirun -np 4 ex25p -o 2 -f 2.0 -rs 1 -rp 1 -prob 4 -m ../data/inline-hex.mesh
 //
+// Device sample runs:
+//               mpirun -np 4 ex25p -o 1 -f 3.0 -rs 3 -rp 1 -prob 2 -pa -d cuda
+//               mpirun -np 4 ex25p -o 2 -f 1.0 -rs 1 -rp 1 -prob 3 -pa -d cuda
+//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -82,24 +86,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLMatrixCoefficient : public MatrixCoefficient
+class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
-   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                              Vector &),
+                            CartesianPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+   virtual void Eval(Vector &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(height, width);
+      K.SetSize(vdim);
       (*Function)(transip, pml, K);
    }
 };
@@ -116,13 +120,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -160,6 +164,8 @@ int main(int argc, char *argv[])
    double freq = 5.0;
    bool herm_conv = true;
    bool visualization = 1;
+   bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -183,12 +189,21 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
 
    if (iprob > 4) { iprob = 4; }
    prob = (prob_type)iprob;
 
-   // 3. Setup the (serial) mesh on all processors.
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 4. Setup the (serial) mesh on all processors.
    if (!mesh_file)
    {
       exact_known = true;
@@ -236,7 +251,7 @@ int main(int argc, char *argv[])
    // Setup PML length
    Array2D<double> length(dim, 2); length = 0.0;
 
-   // 4. Setup the Cartesian PML region.
+   // 5. Setup the Cartesian PML region.
    switch (prob)
    {
       case disc:
@@ -262,13 +277,13 @@ int main(int argc, char *argv[])
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 6. Refine the serial mesh on all processors to increase the resolution.
    for (int l = 0; l < ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh.
+   // 7. Define a parallel mesh by a partitioning of the serial mesh.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -278,13 +293,13 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6a. Reorient mesh in case of a tet mesh
+   // 7a. Reorient mesh in case of a tet mesh
    pmesh->ReorientTetMesh();
 
-   // 7. Set element attributes in order to distinguish elements in the PML
+   // 8. Set element attributes in order to distinguish elements in the PML
    pml->SetAttributes(pmesh);
 
-   // 8. Define a parallel finite element space on the parallel mesh. Here we
+   // 9. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -294,9 +309,9 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 9. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    based on the specific mesh and the problem type.
+   // 10. Determine the list of true (i.e. parallel conforming) essential
+   //     boundary dofs. In this example, the boundary conditions are defined
+   //     based on the specific mesh and the problem type.
    Array<int> ess_tdof_list;
    Array<int> ess_bdr;
    if (pmesh->bdr_attributes.Size())
@@ -336,11 +351,11 @@ int main(int argc, char *argv[])
    }
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 10. Setup Complex Operator convention
+   // 11. Setup Complex Operator convention
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 11. Set up the parallel linear form b(.) which corresponds to the
+   // 12. Set up the parallel linear form b(.) which corresponds to the
    //     right-hand side of the FEM linear system.
    VectorFunctionCoefficient f(dim, source);
    ParComplexLinearForm b(fespace, conv);
@@ -351,7 +366,7 @@ int main(int argc, char *argv[])
    b.Vector::operator=(0.0);
    b.Assemble();
 
-   // 12. Define the solution vector x as a parallel complex finite element grid
+   // 13. Define the solution vector x as a parallel complex finite element grid
    //     function corresponding to fespace.
    ParComplexGridFunction x(fespace);
    x = 0.0;
@@ -359,7 +374,7 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
    x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
 
-   // 13. Set up the parallel sesquilinear form a(.,.)
+   // 14. Set up the parallel sesquilinear form a(.,.)
    //
    //     In Comp
    //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -393,19 +408,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
-   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
+   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
-   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
+   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -413,19 +428,23 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                          new VectorFEMassIntegrator(restr_c2_Im));
 
-   // 14. Assemble the parallel bilinear form and the corresponding linear
+   // 15. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, etc.
+#ifndef MFEM_USE_SUPERLU
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+#endif
    a.Assemble();
 
    OperatorPtr Ah;
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
 
-   // 15. Solve using a direct or an iterative solver
+   // 16. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
    {
+      if (pa) { cout << "PA not available with MFEM_USE_SUPERLU" << endl; }
       // Transform to monolithic HypreParMatrix
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
       SuperLURowLocMatrix SA(*A);
@@ -453,22 +472,20 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
-      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
+      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
 
+      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
       prec.Assemble();
 
-      OperatorPtr PCOpAh;
-      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
       // 16b. Define and apply a parallel GMRES solver for AU=B with a block
       //      diagonal preconditioner based on hypre's AMS preconditioner.
       Array<int> offsets(3);
@@ -477,21 +494,41 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      HypreAMS ams00(*PCOpAh.As<HypreParMatrix>(),fespace);
-      BlockDiagonalPreconditioner BlockAMS(offsets);
-      ScaledOperator ams11(&ams00,
-                           (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
-      BlockAMS.SetDiagonalBlock(0,&ams00);
-      BlockAMS.SetDiagonalBlock(1,&ams11);
+      Operator *pc_r = nullptr;
+      Operator *pc_i = nullptr;
+      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      if (pa)
+      {
+         // Jacobi Smoother
+         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
+         ScaledOperator *d11 = new ScaledOperator(d00, s);
+         pc_r = d00;
+         pc_i = d11;
+      }
+      else
+      {
+         OperatorPtr PCOpAh;
+         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
+         // Hypre AMS
+         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
+         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
+         pc_r = ams00;
+         pc_i = ams11;
+      }
+
+      BlockDiagonalPreconditioner BlockDP(offsets);
+      BlockDP.SetDiagonalBlock(0, pc_r);
+      BlockDP.SetDiagonalBlock(1, pc_i);
 
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetPrintLevel(1);
       gmres.SetKDim(200);
-      gmres.SetMaxIter(2000);
+      gmres.SetMaxIter(pa ? 5000 : 2000);
       gmres.SetRelTol(1e-5);
       gmres.SetAbsTol(0.0);
       gmres.SetOperator(*Ah);
-      gmres.SetPreconditioner(BlockAMS);
+      gmres.SetPreconditioner(BlockDP);
       gmres.Mult(B, X);
    }
 #endif
@@ -819,7 +856,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -830,14 +867,13 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -848,14 +884,13 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -866,14 +901,13 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -887,19 +921,18 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      M = (1.0 / det).real();
+      D = (1.0 / det).real();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -912,19 +945,18 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = (1.0 / det).imag();
+      D = (1.0 / det).imag();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -937,14 +969,13 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = abs(1.0 / det);
+      D = abs(1.0 / det);
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

fem/bilinearform.cpp

@@ -627,33 +627,6 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
       MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
                   "Vector for holding diagonal has wrong size!");
       const Operator *P = fes->GetProlongationMatrix();
-      // For an AMR mesh, a convergent diagonal is assembled with |P^T| d_e,
-      // where |P^T| has the entry-wise absolute values of the conforming
-      // prolongation transpose operator.
-      if (P && !fes->Conforming())
-      {
-         Vector local_diag(P->Height());
-         ext->AssembleDiagonal(local_diag);
-         const SparseMatrix *SP = dynamic_cast<const SparseMatrix*>(P);
-#ifdef MFEM_USE_MPI
-         const HypreParMatrix *HP = dynamic_cast<const HypreParMatrix*>(P);
-#endif
-         if (SP)
-         {
-            SP->AbsMultTranspose(local_diag, diag);
-         }
-#ifdef MFEM_USE_MPI
-         else if (HP)
-         {
-            HP->AbsMultTranspose(1.0, local_diag, 0.0, diag);
-         }
-#endif
-         else
-         {
-            MFEM_ABORT("Prolongation matrix has unexpected type.");
-         }
-         return;
-      }
       if (!IsIdentityProlongation(P))
       {
          Vector local_diag(P->Height());

fem/bilinearform_ext.cpp

@@ -96,9 +96,6 @@ void PABilinearFormExtension::Assemble()
       integrators[i]->AssemblePA(*a->FESpace());
    }
 
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
-
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int intFaceIntegratorCount = intFaceIntegrators.Size();
    for (int i = 0; i < intFaceIntegratorCount; ++i)
@@ -119,7 +116,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
    const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
+   if (elem_restrict)
    {
       localY = 0.0;
       for (int i = 0; i < iSz; ++i)
@@ -323,9 +320,6 @@ void EABilinearFormExtension::Assemble()
               GetTraceElement(0, trialFes->GetMesh()->GetFaceBaseGeometry(0)) ->
               GetDof();
 
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Element assembly does not support AddBoundaryIntegrator yet.");
-
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
    const int intFaceIntegratorCount = intFaceIntegrators.Size();
    if (intFaceIntegratorCount>0)
@@ -800,12 +794,6 @@ void PAMixedBilinearFormExtension::Assemble()
    {
       integrators[i]->AssemblePA(*trialFes, *testFes);
    }
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
-   MFEM_VERIFY(a->GetTFBFI()->Size() == 0,
-               "Partial assembly does not support AddTraceFaceIntegrator yet.");
-   MFEM_VERIFY(a->GetBTFBFI()->Size() == 0,
-               "Partial assembly does not support AddBdrTraceFaceIntegrator yet.");
 }
 
 void PAMixedBilinearFormExtension::Update()

fem/bilininteg.cpp

@@ -1522,6 +1522,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
    double w;
 
 #ifdef MFEM_THREAD_SAFE
+   Vector D;
    DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
    curlshape.SetSize(nd,dimc);
@@ -1529,6 +1530,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
    elmat.SetSize(nd);
    if (MQ) { M.SetSize(dimc); }
+   if (DQ) { D.SetSize(dimc); }
 
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
@@ -1572,6 +1574,12 @@ void CurlCurlIntegrator::AssembleElementMatrix
          Mult(curlshape_dFt, M, curlshape);
          AddMultABt(curlshape, curlshape_dFt, elmat);
       }
+      else if (DQ)
+      {
+         DQ->Eval(D, Trans, ip);
+         D *= w;
+         AddMultADAt(curlshape_dFt, D, elmat);
+      }
       else if (Q)
       {
          w *= Q->Eval(Trans, ip);

fem/bilininteg.hpp

@@ -1961,7 +1961,7 @@ public:
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe);
 
-   void SetupPA(const FiniteElementSpace &fes);
+   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };
 
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -2037,7 +2037,7 @@ public:
                                          const FiniteElement &test_fe,
                                          ElementTransformation &Trans);
 
-   void SetupPA(const FiniteElementSpace &fes);
+   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };
 
 /** Mass integrator (u, v) restricted to the boundary of a domain */
@@ -2300,12 +2300,14 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
    Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
+   Vector D;
    DenseMatrix curlshape, curlshape_dFt, M;
    DenseMatrix vshape, projcurl;
 #endif
 
 protected:
    Coefficient *Q;
+   VectorCoefficient *DQ;
    MatrixCoefficient *MQ;
 
    // PA extension
@@ -2314,12 +2316,17 @@ protected:
    const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
+   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
-   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
    /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }
+   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
 
    /* Given a particular Finite Element, compute the
       element curl-curl matrix elmat */

fem/bilininteg_diffusion_ea.cpp

@@ -130,8 +130,8 @@ static void EADiffusionAssemble2D(const int NE,
 
 template<int T_D1D = 0, int T_Q1D = 0>
 static void EADiffusionAssemble3D(const int NE,
-                                  const Array<double> &b,
                                   const Array<double> &g,
+                                  const Array<double> &b,
                                   const Vector &padata,
                                   Vector &eadata,
                                   const int d1d = 0,

fem/bilininteg_diffusion_pa.cpp

@@ -96,28 +96,26 @@ void PADiffusionSetup2D<2>(const int Q1D,
                            const Vector &c,
                            Vector &d)
 {
+   const int NQ = Q1D*Q1D;
    const bool const_c = c.Size() == 1;
-   const auto W = Reshape(w.Read(), Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
-   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
+   auto D = Reshape(d.Write(), NQ, 3, NE);
+
+   MFEM_FORALL(e, NE,
    {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-            const double c_detJ = W(qx,qy) * coeff / ((J11*J22)-(J21*J12));
-            D(qx,qy,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
-            D(qx,qy,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
-            D(qx,qy,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
-         }
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double coeff = const_c ? C(0,0) : C(q,e);
+         const double c_detJ = W[q] * coeff / ((J11*J22)-(J21*J12));
+         D(q,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
+         D(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
+         D(q,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
       }
    });
 }
@@ -133,35 +131,33 @@ void PADiffusionSetup2D<3>(const int Q1D,
 {
    constexpr int DIM = 2;
    constexpr int SDIM = 3;
+   const int NQ = Q1D*Q1D;
    const bool const_c = c.Size() == 1;
-   const auto W = Reshape(w.Read(), Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,SDIM,DIM,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
-   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
+
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, SDIM, DIM, NE);
+   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
+   auto D = Reshape(d.Write(), NQ, 3, NE);
+   MFEM_FORALL(e, NE,
    {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            const double wq = W(qx,qy);
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J31 = J(qx,qy,2,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double J32 = J(qx,qy,2,1,e);
-            const double E = J11*J11 + J21*J21 + J31*J31;
-            const double G = J12*J12 + J22*J22 + J32*J32;
-            const double F = J11*J12 + J21*J22 + J31*J32;
-            const double iw = 1.0 / sqrt(E*G - F*F);
-            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-            const double alpha = wq * coeff * iw;
-            D(qx,qy,0,e) =  alpha * G; // 1,1
-            D(qx,qy,1,e) = -alpha * F; // 1,2
-            D(qx,qy,2,e) =  alpha * E; // 2,2
-         }
+         const double wq = W[q];
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double E = J11*J11 + J21*J21 + J31*J31;
+         const double G = J12*J12 + J22*J22 + J32*J32;
+         const double F = J11*J12 + J21*J22 + J31*J32;
+         const double iw = 1.0 / sqrt(E*G - F*F);
+         const double coeff = const_c ? C(0,0) : C(q,e);
+         const double alpha = wq * coeff * iw;
+         D(q,0,e) =  alpha * G; // 1,1
+         D(q,1,e) = -alpha * F; // 1,2
+         D(q,2,e) =  alpha * E; // 2,2
       }
    });
 }
@@ -174,53 +170,47 @@ static void PADiffusionSetup3D(const int Q1D,
                                const Vector &c,
                                Vector &d)
 {
+   const int NQ = Q1D*Q1D*Q1D;
    const bool const_c = c.Size() == 1;
-   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, 6, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
+   auto D = Reshape(d.Write(), NQ, 6, NE);
+   MFEM_FORALL(e, NE,
    {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               const double J11 = J(qx,qy,qz,0,0,e);
-               const double J21 = J(qx,qy,qz,1,0,e);
-               const double J31 = J(qx,qy,qz,2,0,e);
-               const double J12 = J(qx,qy,qz,0,1,e);
-               const double J22 = J(qx,qy,qz,1,1,e);
-               const double J32 = J(qx,qy,qz,2,1,e);
-               const double J13 = J(qx,qy,qz,0,2,e);
-               const double J23 = J(qx,qy,qz,1,2,e);
-               const double J33 = J(qx,qy,qz,2,2,e);
-               const double detJ = J11 * (J22 * J33 - J32 * J23) -
-               /* */               J21 * (J12 * J33 - J32 * J13) +
-               /* */               J31 * (J12 * J23 - J22 * J13);
-               const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-               const double c_detJ = W(qx,qy,qz) * coeff / detJ;
-               // adj(J)
-               const double A11 = (J22 * J33) - (J23 * J32);
-               const double A12 = (J32 * J13) - (J12 * J33);
-               const double A13 = (J12 * J23) - (J22 * J13);
-               const double A21 = (J31 * J23) - (J21 * J33);
-               const double A22 = (J11 * J33) - (J13 * J31);
-               const double A23 = (J21 * J13) - (J11 * J23);
-               const double A31 = (J21 * J32) - (J31 * J22);
-               const double A32 = (J31 * J12) - (J11 * J32);
-               const double A33 = (J11 * J22) - (J12 * J21);
-               // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-               D(qx,qy,qz,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-               D(qx,qy,qz,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-               D(qx,qy,qz,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-               D(qx,qy,qz,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-               D(qx,qy,qz,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-               D(qx,qy,qz,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
-            }
-         }
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double detJ = J11 * (J22 * J33 - J32 * J23) -
+         /* */               J21 * (J12 * J33 - J32 * J13) +
+         /* */               J31 * (J12 * J23 - J22 * J13);
+         const double coeff = const_c ? C(0,0) : C(q,e);
+         const double c_detJ = W[q] * coeff / detJ;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+         // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+         D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+         D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+         D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+         D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+         D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+         D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
       }
    });
 }
@@ -263,7 +253,8 @@ static void PADiffusionSetup(const int dim,
    }
 }
 
-void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
+void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
+                                  const bool force)
 {
    // Assuming the same element type
    fespace = &fes;
@@ -272,7 +263,7 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
    const FiniteElement &el = *fes.GetFE(0);
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
 #ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
+   if (DeviceCanUseCeed() && !force)
    {
       if (ceedDataPtr) { delete ceedDataPtr; }
       CeedData* ptr = new CeedData();
@@ -280,6 +271,8 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
       InitCeedCoeff(Q, ptr);
       return CeedPADiffusionAssemble(fes, *ir, *ptr);
    }
+#else
+   MFEM_CONTRACT_VAR(force);
 #endif
    const int dims = el.GetDim();
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
@@ -756,17 +749,9 @@ static void PADiffusionAssembleDiagonal(const int dim,
 
 void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonalPA(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
-                                  maps->B, maps->G, pa_data, diag);
-   }
+   if (pa_data.Size()==0) { SetupPA(*fespace, true); }
+   PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
+                               maps->B, maps->G, pa_data, diag);
 }
 
 
@@ -1723,7 +1708,29 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 #ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      CeedAddMultPA(ceedDataPtr, x, y);
+      const CeedScalar *x_ptr;
+      CeedScalar *y_ptr;
+      CeedMemType mem;
+      CeedGetPreferredMemType(internal::ceed, &mem);
+      if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
+      {
+         x_ptr = x.Read();
+         y_ptr = y.ReadWrite();
+      }
+      else
+      {
+         x_ptr = x.HostRead();
+         y_ptr = y.HostReadWrite();
+         mem = CEED_MEM_HOST;
+      }
+      CeedVectorSetArray(ceedDataPtr->u, mem, CEED_USE_POINTER,
+                         const_cast<CeedScalar*>(x_ptr));
+      CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, y_ptr);
+
+      CeedOperatorApplyAdd(ceedDataPtr->oper, ceedDataPtr->u, ceedDataPtr->v,
+                           CEED_REQUEST_IMMEDIATE);
+
+      CeedVectorSyncArray(ceedDataPtr->v, mem);
    }
    else
 #endif

fem/bilininteg_hcurl.cpp

@@ -678,10 +678,12 @@ static void PACurlCurlSetup3D(const int Q1D,
                               Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
+   const bool symmetric = (coeffDim != 9);
    auto W = w.Read();
    auto J = Reshape(j.Read(), NQ, 3, 3, NE);
    auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), NQ, 6, NE);
+   auto y = Reshape(op.Write(), NQ, symmetric ? 6 : 9, NE);
+
    MFEM_FORALL(e, NE,
    {
       for (int q = 0; q < NQ; ++q)
@@ -699,19 +701,69 @@ static void PACurlCurlSetup3D(const int Q1D,
          /* */               J21 * (J12 * J33 - J32 * J13) +
          /* */               J31 * (J12 * J23 - J22 * J13);
 
-         const double D1 = coeff(0, q, e);
-         const double D2 = coeffDim == 3 ? coeff(1, q, e) : D1;
-         const double D3 = coeffDim == 3 ? coeff(2, q, e) : D1;
-
-         // set y to the 6 entries of J^T D J / det^2
          const double c_detJ = W[q] / detJ;
 
-         y(q,0,e) = c_detJ * (D1*J11*J11 + D2*J21*J21 + D3*J31*J31); // 1,1
-         y(q,1,e) = c_detJ * (D1*J11*J12 + D2*J21*J22 + D3*J31*J32); // 1,2
-         y(q,2,e) = c_detJ * (D1*J11*J13 + D2*J21*J23 + D3*J31*J33); // 1,3
-         y(q,3,e) = c_detJ * (D1*J12*J12 + D2*J22*J22 + D3*J32*J32); // 2,2
-         y(q,4,e) = c_detJ * (D1*J12*J13 + D2*J22*J23 + D3*J32*J33); // 2,3
-         y(q,5,e) = c_detJ * (D1*J13*J13 + D2*J23*J23 + D3*J33*J33); // 3,3
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+         {
+            // Set y to the 6 or 9 entries of J^T M J / det
+            const double M11 = coeff(0, q, e);
+            const double M12 = coeff(1, q, e);
+            const double M13 = coeff(2, q, e);
+            const double M21 = (!symmetric) ? coeff(3, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(4, q, e) : coeff(3, q, e);
+            const double M23 = (!symmetric) ? coeff(5, q, e) : coeff(4, q, e);
+            const double M31 = (!symmetric) ? coeff(6, q, e) : M13;
+            const double M32 = (!symmetric) ? coeff(7, q, e) : M23;
+            const double M33 = (!symmetric) ? coeff(8, q, e) : coeff(5, q, e);
+
+            // First compute R = MJ
+            const double R11 = M11*J11 + M12*J21 + M13*J31;
+            const double R12 = M11*J12 + M12*J22 + M13*J32;
+            const double R13 = M11*J13 + M12*J23 + M13*J33;
+            const double R21 = M21*J11 + M22*J21 + M23*J31;
+            const double R22 = M21*J12 + M22*J22 + M23*J32;
+            const double R23 = M21*J13 + M22*J23 + M23*J33;
+            const double R31 = M31*J11 + M32*J21 + M33*J31;
+            const double R32 = M31*J12 + M32*J22 + M33*J32;
+            const double R33 = M31*J13 + M32*J23 + M33*J33;
+
+            // Now set y to J^T R / det
+            y(q,0,e) = c_detJ * (J11*R11 + J21*R21 + J31*R31); // 1,1
+            const double Y12 = c_detJ * (J11*R12 + J21*R22 + J31*R32);
+            y(q,1,e) = Y12; // 1,2
+            y(q,2,e) = c_detJ * (J11*R13 + J21*R23 + J31*R33); // 1,3
+
+            const double Y21 = c_detJ * (J12*R11 + J22*R21 + J32*R31);
+            const double Y22 = c_detJ * (J12*R12 + J22*R22 + J32*R32);
+            const double Y23 = c_detJ * (J12*R13 + J22*R23 + J32*R33);
+
+            const double Y33 = c_detJ * (J13*R13 + J23*R23 + J33*R33);
+
+            y(q,3,e) = symmetric ? Y22 : Y21; // 2,2 or 2,1
+            y(q,4,e) = symmetric ? Y23 : Y22; // 2,3 or 2,2
+            y(q,5,e) = symmetric ? Y33 : Y23; // 3,3 or 2,3
+
+            if (!symmetric)
+            {
+               y(q,6,e) = c_detJ * (J13*R11 + J23*R21 + J33*R31); // 3,1
+               y(q,7,e) = c_detJ * (J13*R12 + J23*R22 + J33*R32); // 3,2
+               y(q,8,e) = Y33; // 3,3
+            }
+         }
+         else  // Vector or scalar coefficient version
+         {
+            // Set y to the 6 entries of J^T D J / det^2
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeffDim == 3 ? coeff(1, q, e) : D1;
+            const double D3 = coeffDim == 3 ? coeff(2, q, e) : D1;
+
+            y(q,0,e) = c_detJ * (D1*J11*J11 + D2*J21*J21 + D3*J31*J31); // 1,1
+            y(q,1,e) = c_detJ * (D1*J11*J12 + D2*J21*J22 + D3*J31*J32); // 1,2
+            y(q,2,e) = c_detJ * (D1*J11*J13 + D2*J21*J23 + D3*J31*J33); // 1,3
+            y(q,3,e) = c_detJ * (D1*J12*J12 + D2*J22*J22 + D3*J32*J32); // 2,2
+            y(q,4,e) = c_detJ * (D1*J12*J13 + D2*J22*J23 + D3*J32*J33); // 2,3
+            y(q,5,e) = c_detJ * (D1*J13*J13 + D2*J23*J23 + D3*J33*J33); // 3,3
+         }
       }
    });
 }
@@ -736,6 +788,8 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
+   const int dimc = (dim == 3) ? 3 : 1;
+
    ne = fes.GetNE();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -745,36 +799,103 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   const int ndata = (dim == 2) ? 1 : 6;
+   const int MQsymmDim = MQ ? (MQ->GetWidth() * (MQ->GetWidth() + 1)) / 2 : 0;
+   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
+   const int MQdim = MQ ? (MQ->IsSymmetric() ? MQsymmDim : MQfullDim) : 0;
+   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
+
+   symmetric = MQ ? MQ->IsSymmetric() : true;
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
    pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
 
-   Vector coeff(ne * nq);
+   Vector coeff(coeffDim * ne * nq);
    coeff = 1.0;
-   if (Q)
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ || MQ)
    {
+      Vector D(DQ ? coeffDim : 0);
+      DenseMatrix M;
+      Vector Msymm;
+      if (MQ)
+      {
+         if (symmetric)
+         {
+            Msymm.SetSize(MQsymmDim);
+         }
+         else
+         {
+            M.SetSize(dimc);
+         }
+      }
+
+      if (DQ)
+      {
+         MFEM_VERIFY(coeffDim == dimc, "");
+      }
+      if (MQ)
+      {
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
+      }
+
       for (int e=0; e<ne; ++e)
       {
          ElementTransformation *tr = mesh->GetElementTransformation(e);
          for (int p=0; p<nq; ++p)
          {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+            if (MQ)
+            {
+               if (MQ->IsSymmetric())
+               {
+                  MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<MQsymmDim; ++i)
+                  {
+                     coeffh(i, p, e) = Msymm[i];
+                  }
+               }
+               else
+               {
+                  MQ->Eval(M, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<dimc; ++i)
+                     for (int j=0; j<dimc; ++j)
+                     {
+                        coeffh(j+(i*dimc), p, e) = M(i,j);
+                     }
+               }
+            }
+            else if (DQ)
+            {
+               DQ->Eval(D, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = D[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
          }
       }
    }
 
-   if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
+   if (el->GetDerivType() != mfem::FiniteElement::CURL)
    {
-      PACurlCurlSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
-                        coeff, pa_data);
+      MFEM_ABORT("Unknown kernel.");
    }
-   else if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
+
+   if (dim == 3)
    {
-      PACurlCurlSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
-                        coeff, pa_data);
+      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
+                        pa_data);
    }
    else
    {
-      MFEM_ABORT("Unknown kernel.");
+      PACurlCurlSetup2D(quad1D, ne, ir->GetWeights(), geom->J, coeff, pa_data);
    }
 }
 
@@ -901,6 +1022,7 @@ static void PACurlCurlApply2D(const int D1D,
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PACurlCurlApply3D(const int D1D,
                               const int Q1D,
+                              const bool symmetric,
                               const int NE,
                               const Array<double> &_Bo,
                               const Array<double> &_Bc,
@@ -928,7 +1050,7 @@ static void PACurlCurlApply3D(const int D1D,
    auto Bct = Reshape(_Bct.Read(), D1D, Q1D);
    auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
    auto Gct = Reshape(_Gct.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, (symmetric ? 6 : 9), NE);
    auto x = Reshape(_x.Read(), 3*(D1D-1)*D1D*D1D, NE);
    auto y = Reshape(_y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
@@ -1171,15 +1293,18 @@ static void PACurlCurlApply3D(const int D1D,
                const double O11 = op(qx,qy,qz,0,e);
                const double O12 = op(qx,qy,qz,1,e);
                const double O13 = op(qx,qy,qz,2,e);
-               const double O22 = op(qx,qy,qz,3,e);
-               const double O23 = op(qx,qy,qz,4,e);
-               const double O33 = op(qx,qy,qz,5,e);
+               const double O21 = symmetric ? O12 : op(qx,qy,qz,3,e);
+               const double O22 = symmetric ? op(qx,qy,qz,3,e) : op(qx,qy,qz,4,e);
+               const double O23 = symmetric ? op(qx,qy,qz,4,e) : op(qx,qy,qz,5,e);
+               const double O31 = symmetric ? O13 : op(qx,qy,qz,6,e);
+               const double O32 = symmetric ? O23 : op(qx,qy,qz,7,e);
+               const double O33 = symmetric ? op(qx,qy,qz,5,e) : op(qx,qy,qz,8,e);
 
                const double c1 = (O11 * curl[qz][qy][qx][0]) + (O12 * curl[qz][qy][qx][1]) +
                                  (O13 * curl[qz][qy][qx][2]);
-               const double c2 = (O12 * curl[qz][qy][qx][0]) + (O22 * curl[qz][qy][qx][1]) +
+               const double c2 = (O21 * curl[qz][qy][qx][0]) + (O22 * curl[qz][qy][qx][1]) +
                                  (O23 * curl[qz][qy][qx][2]);
-               const double c3 = (O13 * curl[qz][qy][qx][0]) + (O23 * curl[qz][qy][qx][1]) +
+               const double c3 = (O31 * curl[qz][qy][qx][0]) + (O32 * curl[qz][qy][qx][1]) +
                                  (O33 * curl[qz][qy][qx][2]);
 
                curl[qz][qy][qx][0] = c1;
@@ -1410,7 +1535,7 @@ void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
    if (dim == 3)
    {
-      PACurlCurlApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
+      PACurlCurlApply3D(dofs1D, quad1D, symmetric, ne, mapsO->B, mapsC->B, mapsO->Bt,
                         mapsC->Bt, mapsC->G, mapsC->Gt, pa_data, x, y);
    }
    else if (dim == 2)

fem/bilininteg_mass_pa.cpp

@@ -23,7 +23,7 @@ namespace mfem
 
 // PA Mass Assemble kernel
 
-void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
+void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
 {
    // Assuming the same element type
    fespace = &fes;
@@ -33,7 +33,7 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    ElementTransformation *T = mesh->GetElementTransformation(0);
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
 #ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
+   if (DeviceCanUseCeed() && !force)
    {
       if (ceedDataPtr) { delete ceedDataPtr; }
       CeedData* ptr = new CeedData();
@@ -41,6 +41,8 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
       InitCeedCoeff(Q, ptr);
       return CeedPAMassAssemble(fes, *ir, *ptr);
    }
+#else
+   MFEM_CONTRACT_VAR(force);
 #endif
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
@@ -92,64 +94,49 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
    if (dim==2)
    {
       const int NE = ne;
-      const int Q1D = quad1D;
+      const int NQ = nq;
       const bool const_c = coeff.Size() == 1;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
-      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
-      MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
+      auto w = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
+      auto C =
+         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
+      auto v = Reshape(pa_data.Write(), NQ, NE);
+      MFEM_FORALL(e, NE,
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               const double J11 = J(qx,qy,0,0,e);
-               const double J12 = J(qx,qy,1,0,e);
-               const double J21 = J(qx,qy,0,1,e);
-               const double J22 = J(qx,qy,1,1,e);
-               const double detJ = (J11*J22)-(J21*J12);
-               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-               v(qx,qy,e) =  W(qx,qy) * coeff * detJ;
-            }
+            const double J11 = J(q,0,0,e);
+            const double J12 = J(q,1,0,e);
+            const double J21 = J(q,0,1,e);
+            const double J22 = J(q,1,1,e);
+            const double detJ = (J11*J22)-(J21*J12);
+            const double coeff = const_c ? C(0,0) : C(q,e);
+            v(q,e) =  w[q] * coeff * detJ;
          }
       });
    }
    if (dim==3)
    {
       const int NE = ne;
-      const int Q1D = quad1D;
+      const int NQ = nq;
       const bool const_c = coeff.Size() == 1;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
-      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
-      MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+      auto W = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
+      auto C =
+         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
+      auto v = Reshape(pa_data.Write(), NQ,NE);
+      MFEM_FORALL(e, NE,
       {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qz,z,Q1D)
-               {
-                  const double J11 = J(qx,qy,qz,0,0,e);
-                  const double J21 = J(qx,qy,qz,1,0,e);
-                  const double J31 = J(qx,qy,qz,2,0,e);
-                  const double J12 = J(qx,qy,qz,0,1,e);
-                  const double J22 = J(qx,qy,qz,1,1,e);
-                  const double J32 = J(qx,qy,qz,2,1,e);
-                  const double J13 = J(qx,qy,qz,0,2,e);
-                  const double J23 = J(qx,qy,qz,1,2,e);
-                  const double J33 = J(qx,qy,qz,2,2,e);
-                  const double detJ = J11 * (J22 * J33 - J32 * J23) -
-                  /* */               J21 * (J12 * J33 - J32 * J13) +
-                  /* */               J31 * (J12 * J23 - J22 * J13);
-                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * detJ;
-               }
-            }
+            const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e);
+            const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e);
+            const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e);
+            const double detJ = J11 * (J22 * J33 - J32 * J23) -
+            /* */               J21 * (J12 * J33 - J32 * J13) +
+            /* */               J31 * (J12 * J23 - J22 * J13);
+            const double coeff = const_c ? C(0,0) : C(q,e);
+            v(q,e) = W[q] * coeff * detJ;
          }
       });
    }
@@ -468,16 +455,8 @@ static void PAMassAssembleDiagonal(const int dim, const int D1D,
 
 void MassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonalPA(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
+   if (pa_data.Size()==0) { SetupPA(*fespace, true); }
+   PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
 }
 
 
@@ -1230,7 +1209,29 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 #ifdef MFEM_USE_CEED
    if (DeviceCanUseCeed())
    {
-      CeedAddMultPA(ceedDataPtr, x, y);
+      const CeedScalar *x_ptr;
+      CeedScalar *y_ptr;
+      CeedMemType mem;
+      CeedGetPreferredMemType(internal::ceed, &mem);
+      if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
+      {
+         x_ptr = x.Read();
+         y_ptr = y.ReadWrite();
+      }
+      else
+      {
+         x_ptr = x.HostRead();
+         y_ptr = y.HostReadWrite();
+         mem = CEED_MEM_HOST;
+      }
+      CeedVectorSetArray(ceedDataPtr->u, mem, CEED_USE_POINTER,
+                         const_cast<CeedScalar*>(x_ptr));
+      CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, y_ptr);
+
+      CeedOperatorApplyAdd(ceedDataPtr->oper, ceedDataPtr->u, ceedDataPtr->v,
+                           CEED_REQUEST_IMMEDIATE);
+
+      CeedVectorSyncArray(ceedDataPtr->v, mem);
    }
    else
 #endif

fem/bilininteg_vectorfe.cpp

@@ -746,12 +746,10 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    symmetric = MQ ? MQ->IsSymmetric() : true;
 
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
-
-   if ((trial_curl && test_div) || (trial_div && test_curl))
+   if ((trial_fetype == mfem::FiniteElement::CURL &&
+        test_fetype == mfem::FiniteElement::DIV) ||
+       (trial_fetype == mfem::FiniteElement::DIV &&
+        test_fetype == mfem::FiniteElement::CURL))
       pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
                       Device::GetMemoryType());
    else
@@ -831,27 +829,34 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       }
    }
 
-   if (trial_curl && test_curl && dim == 3)
+   if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype
+       && dim == 3)
    {
       PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
-   else if (trial_curl && test_curl && dim == 2)
+   else if (trial_fetype == mfem::FiniteElement::CURL
+            && test_fetype == trial_fetype && dim == 2)
    {
       PAHcurlSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
-   else if (trial_div && test_div && dim == 3)
+   else if (trial_fetype == mfem::FiniteElement::DIV
+            && test_fetype == trial_fetype && dim == 3)
    {
       PAHdivSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
-   else if (trial_div && test_div && dim == 2)
+   else if (trial_fetype == mfem::FiniteElement::DIV
+            && test_fetype == trial_fetype && dim == 2)
    {
       PAHdivSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
-   else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
+   else if (((trial_fetype == mfem::FiniteElement::CURL &&
+              test_fetype == mfem::FiniteElement::DIV) ||
+             (trial_fetype == mfem::FiniteElement::DIV &&
+              test_fetype == mfem::FiniteElement::CURL)) &&
             test_fel->GetOrder() == trial_fel->GetOrder())
    {
       if (coeffDim == 1)
@@ -860,7 +865,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
       }
       else
       {
-         const bool tr = (trial_div && test_curl);
+         const bool tr = (trial_fetype == mfem::FiniteElement::DIV &&
+                          test_fetype == mfem::FiniteElement::CURL);
          if (dim == 3)
             PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
                                geom->J, coeff, pa_data);
@@ -917,31 +923,29 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
-
    if (dim == 3)
    {
-      if (trial_curl && test_curl)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
          PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
                             mapsO->Bt, mapsC->Bt, pa_data, x, y);
       }
-      else if (trial_div && test_div)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                            mapsC->Bt, pa_data, x, y);
       }
-      else if (trial_curl && test_div)
+      else if (trial_fetype == mfem::FiniteElement::CURL &&
+               test_fetype == mfem::FiniteElement::DIV)
       {
          const bool scalarCoeff = !(VQ || MQ);
          PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
                                 true, mapsO->B, mapsC->B, mapsOtest->Bt,
                                 mapsCtest->Bt, pa_data, x, y);
       }
-      else if (trial_div && test_curl)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == mfem::FiniteElement::CURL)
       {
          const bool scalarCoeff = !(VQ || MQ);
          PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
@@ -955,21 +959,26 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      if (trial_curl && test_curl)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
          PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
                             mapsO->Bt, mapsC->Bt, pa_data, x, y);
       }
-      else if (trial_div && test_div)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                            mapsC->Bt, pa_data, x, y);
       }
-      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      else if ((trial_fetype == mfem::FiniteElement::CURL &&
+                test_fetype == mfem::FiniteElement::DIV) ||
+               (trial_fetype == mfem::FiniteElement::DIV &&
+                test_fetype == mfem::FiniteElement::CURL))
       {
          const bool scalarCoeff = !(VQ || MQ);
+         const bool trialHcurl = (trial_fetype == mfem::FiniteElement::CURL);
          PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                trial_curl, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                trialHcurl, mapsO->B, mapsC->B, mapsOtest->Bt,
                                 mapsCtest->Bt, pa_data, x, y);
       }
       else

fem/complex_fem.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "complex_fem.hpp"
+#include "../general/forall.hpp"
 
 using namespace std;
 
@@ -19,16 +20,21 @@ namespace mfem
 ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
    : Vector(2*(fes->GetVSize()))
 {
-   gfr = new GridFunction(fes, data);
-   gfi = new GridFunction(fes, &data[fes->GetVSize()]);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   gfr = new GridFunction();
+   gfr->MakeRef(fes, *this, 0);
+
+   gfi = new GridFunction();
+   gfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 void
 ComplexGridFunction::Update()
 {
-   FiniteElementSpace * fes = gfr->FESpace();
-
-   int vsize = fes->GetVSize();
+   FiniteElementSpace *fes = gfr->FESpace();
+   const int vsize = fes->GetVSize();
 
    const Operator *T = fes->GetUpdateOperator();
    if (T)
@@ -40,30 +46,36 @@ ComplexGridFunction::Update()
 
       // Our data array now contains old data as well as being the wrong size so
       // reallocate it.
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r(data, vsize);
-      Vector gf_i((data) ? &data[vsize] : data, vsize);
+      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
+      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
 
       // Copy the updated GridFunctions into the new data array
       gf_r = *gfr;
       gf_i = *gfi;
+      gf_r.SyncAliasMemory(*this);
+      gf_i.SyncAliasMemory(*this);
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      gfr->NewDataAndSize(data, vsize);
-      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      gfr->MakeRef(*this, 0, vsize);
+      gfi->MakeRef(*this, vsize, vsize);
    }
    else
    {
       // The existing data will not be transferred to the new GridFunctions so
-      // delete it a allocate a new array
+      // delete it and allocate a new array
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Point the individual GridFunctions to the new data array
-      gfr->NewDataAndSize(data, vsize);
-      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      gfr->MakeRef(*this, 0, vsize);
+      gfi->MakeRef(*this, vsize, vsize);
 
       // These updates will only set the proper 'sequence' value within the
       // individual GridFunction objects because their sizes are already correct
@@ -76,16 +88,24 @@ void
 ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                         Coefficient &imag_coeff)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectCoefficient(real_coeff);
    gfi->ProjectCoefficient(imag_coeff);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
 ComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                         VectorCoefficient &imag_vcoeff)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectCoefficient(real_vcoeff);
    gfi->ProjectCoefficient(imag_vcoeff);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -93,8 +113,12 @@ ComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff,
                                            Array<int> &attr)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficient(real_coeff, attr);
    gfi->ProjectBdrCoefficient(imag_coeff, attr);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -102,8 +126,12 @@ ComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient &real_vcoeff,
                                                  VectorCoefficient &imag_vcoeff,
                                                  Array<int> &attr)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
    gfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 void
@@ -113,18 +141,28 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                   &imag_vcoeff,
                                                   Array<int> &attr)
 {
+   gfr->SyncMemory(*this);
+   gfi->SyncMemory(*this);
    gfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
    gfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
+   gfr->SyncAliasMemory(*this);
+   gfi->SyncAliasMemory(*this);
 }
 
 
-ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *f,
+ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
                                      ComplexOperator::Convention convention)
-   : Vector(2*(f->GetVSize())),
+   : Vector(2*(fes->GetVSize())),
      conv(convention)
 {
-   lfr = new LinearForm(f, data);
-   lfi = new LinearForm(f, &data[f->GetVSize()]);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   lfr = new LinearForm();
+   lfr->MakeRef(fes, *this, 0);
+
+   lfi = new LinearForm();
+   lfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
@@ -133,8 +171,14 @@ ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
    : Vector(2*(fes->GetVSize())),
      conv(convention)
 {
-   lfr = new LinearForm(fes, lf_r);  lfr->SetData(data);
-   lfi = new LinearForm(fes, lf_i);  lfi->SetData(&data[fes->GetVSize()]);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   lfr = new LinearForm(fes, lf_r);
+   lfi = new LinearForm(fes, lf_i);
+
+   lfr->MakeRef(fes, *this, 0);
+   lfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 ComplexLinearForm::~ComplexLinearForm()
@@ -189,42 +233,43 @@ void
 ComplexLinearForm::Update()
 {
    FiniteElementSpace *fes = lfr->FESpace();
-
    this->Update(fes);
 }
 
 void
 ComplexLinearForm::Update(FiniteElementSpace *fes)
 {
-   int vsize = fes->GetVSize();
-   SetSize(2 * vsize);
+   UseDevice(true);
+   SetSize(2 * fes->GetVSize());
+   this->Vector::operator=(0.0);
 
-   Vector vlfr(data, vsize);
-   Vector vlfi((data) ? &data[vsize] : data, vsize);
-
-   lfr->Update(fes, vlfr, 0);
-   lfi->Update(fes, vlfi, 0);
+   lfr->MakeRef(fes, *this, 0);
+   lfi->MakeRef(fes, *this, fes->GetVSize());
 }
 
 void
 ComplexLinearForm::Assemble()
 {
+   lfr->SyncMemory(*this);
+   lfi->SyncMemory(*this);
    lfr->Assemble();
    lfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      *lfi *= -1.0;
-   }
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *lfi *= -1.0; }
+   lfr->SyncAliasMemory(*this);
+   lfi->SyncAliasMemory(*this);
 }
 
 complex<double>
 ComplexLinearForm::operator()(const ComplexGridFunction &gf) const
 {
-   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
+   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
+   lfr->SyncMemory(*this);
+   lfi->SyncMemory(*this);
    return complex<double>((*lfr)(gf.real()) - s * (*lfi)(gf.imag()),
                           (*lfr)(gf.imag()) + s * (*lfi)(gf.real()));
 }
 
+
 bool SesquilinearForm::RealInteg()
 {
    int nint = blfr->GetFBFI()->Size() + blfr->GetDBFI()->Size() +
@@ -341,34 +386,45 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                    Vector &X, Vector &B,
                                    int ci)
 {
-   FiniteElementSpace * fes = blfr->FESpace();
-   int vsize  = fes->GetVSize();
+   FiniteElementSpace *fes = blfr->FESpace();
+   const int vsize = fes->GetVSize();
 
-   // Allocate temporary vectors
-   Vector b_0(vsize); b_0 = 0.0;
+   // Allocate temporary vector
+   Vector b_0;
+   b_0.UseDevice(true);
+   b_0.SetSize(vsize);
+   b_0 = 0.0;
 
    // Extract the real and imaginary parts of the input vectors
    MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
+   x.Read();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
 
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   Vector b_r(b.GetData(), vsize);
-   Vector b_i(&(b.GetData())[vsize], vsize);
+   b.Read();
+   Vector b_r; b_r.MakeRef(b, 0, vsize);
+   Vector b_i; b_i.MakeRef(b, vsize, vsize);
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
 
-   int tvsize = fes->GetTrueVSize();
+   const int tvsize = fes->GetTrueVSize();
    OperatorHandle A_r, A_i;
 
+   X.UseDevice(true);
    X.SetSize(2 * tvsize);
-   B.SetSize(2 * tvsize);
+   X = 0.0;
 
-   Vector X_0(tvsize), B_0(tvsize);
-   Vector X_r(X.GetData(),tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
-   Vector B_r(B.GetData(), tvsize);
-   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   B.UseDevice(true);
+   B.SetSize(2 * tvsize);
+   B = 0.0;
+
+   Vector X_r; X_r.MakeRef(X, 0, tvsize);
+   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
+   Vector B_r; B_r.MakeRef(B, 0, tvsize);
+   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
+
+   Vector X_0, B_0;
 
    if (RealInteg())
    {
@@ -418,13 +474,18 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       // conform with standard essential BC treatment
       if (A_i.Is<ConstrainedOperator>())
       {
-         int n = ess_tdof_list.Size();
-         for (int k = 0; k < n; k++)
+         const int n = ess_tdof_list.Size();
+         auto d_B_r = B_r.Write();
+         auto d_B_i = B_i.Write();
+         auto d_X_r = X_r.Read();
+         auto d_X_i = X_i.Read();
+         auto d_idx = ess_tdof_list.Read();
+         MFEM_FORALL(i, n,
          {
-            int j = ess_tdof_list[k];
-            B_r(j) = X_r(j);
-            B_i(j) = X_i(j);
-         }
+            const int j = d_idx[i];
+            d_B_r[j] = d_X_r[j];
+            d_B_i[j] = d_X_i[j];
+         });
          A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
          (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
       }
@@ -436,6 +497,16 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       b_i *= -1.0;
    }
 
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
+   b_r.SyncAliasMemory(b);
+   b_i.SyncAliasMemory(b);
+
+   X_r.SyncAliasMemory(X);
+   X_i.SyncAliasMemory(X);
+   B_r.SyncAliasMemory(B);
+   B_i.SyncAliasMemory(B);
+
    // A = A_r + i A_i
    A.Clear();
    if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
@@ -528,29 +599,32 @@ void
 SesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                      Vector &x)
 {
-   FiniteElementSpace * fes = blfr->FESpace();
+   FiniteElementSpace *fes = blfr->FESpace();
 
    const SparseMatrix *P = fes->GetConformingProlongation();
-
-   int vsize  = fes->GetVSize();
-   int tvsize = X.Size() / 2;
-
-   Vector X_r(X.GetData(), tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
-
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
-
    if (!P)
    {
       x = X;
+      return;
    }
-   else
-   {
-      // Apply conforming prolongation
-      P->Mult(X_r, x_r);
-      P->Mult(X_i, x_i);
-   }
+
+   const int vsize  = fes->GetVSize();
+   const int tvsize = X.Size() / 2;
+
+   X.Read();
+   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
+   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
+
+   x.Write();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+
+   // Apply conforming prolongation
+   P->Mult(X_r, x_r);
+   P->Mult(X_i, x_i);
+
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
 }
 
 void
@@ -566,16 +640,21 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
    : Vector(2*(pfes->GetVSize()))
 {
-   pgfr = new ParGridFunction(pfes, data);
-   pgfi = new ParGridFunction(pfes, (data) ? &data[pfes->GetVSize()]:data);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
+
+   pgfr = new ParGridFunction();
+   pgfr->MakeRef(pfes, *this, 0);
+
+   pgfi = new ParGridFunction();
+   pgfi->MakeRef(pfes, *this, pfes->GetVSize());
 }
 
 void
 ParComplexGridFunction::Update()
 {
-   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
-
-   int vsize = pfes->GetVSize();
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
+   const int vsize = pfes->GetVSize();
 
    const Operator *T = pfes->GetUpdateOperator();
    if (T)
@@ -587,30 +666,34 @@ ParComplexGridFunction::Update()
 
       // Our data array now contains old data as well as being the wrong size so
       // reallocate it.
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Create temporary vectors which point to the new data array
-      Vector gf_r(data, vsize);
-      Vector gf_i((data) ? &data[vsize] : data, vsize);
+      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
+      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
 
       // Copy the updated GridFunctions into the new data array
-      gf_r = *pgfr;
-      gf_i = *pgfi;
+      gf_r = *pgfr; gf_r.SyncAliasMemory(*this);
+      gf_i = *pgfi; gf_i.SyncAliasMemory(*this);
 
       // Replace the individual data arrays with pointers into the new data
       // array
-      pgfr->NewDataAndSize(data, vsize);
-      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      pgfr->MakeRef(*this, 0, vsize);
+      pgfi->MakeRef(*this, vsize, vsize);
    }
    else
    {
       // The existing data will not be transferred to the new GridFunctions so
-      // delete it a allocate a new array
+      // delete it and allocate a new array
+      UseDevice(true);
       this->SetSize(2 * vsize);
+      this->Vector::operator=(0.0);
 
       // Point the individual GridFunctions to the new data array
-      pgfr->NewDataAndSize(data, vsize);
-      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
+      pgfr->MakeRef(*this, 0, vsize);
+      pgfi->MakeRef(*this, vsize, vsize);
 
       // These updates will only set the proper 'sequence' value within the
       // individual GridFunction objects because their sizes are already correct
@@ -623,16 +706,24 @@ void
 ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectCoefficient(real_coeff);
    pgfi->ProjectCoefficient(imag_coeff);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                            VectorCoefficient &imag_vcoeff)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectCoefficient(real_vcoeff);
    pgfi->ProjectCoefficient(imag_vcoeff);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -640,8 +731,12 @@ ParComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                               Coefficient &imag_coeff,
                                               Array<int> &attr)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficient(real_coeff, attr);
    pgfi->ProjectBdrCoefficient(imag_coeff, attr);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -651,8 +746,12 @@ ParComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient
                                                     &imag_vcoeff,
                                                     Array<int> &attr)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
    pgfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
@@ -662,36 +761,51 @@ ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                      &imag_vcoeff,
                                                      Array<int> &attr)
 {
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
    pgfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::Distribute(const Vector *tv)
 {
-   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
-   HYPRE_Int size = pfes->GetTrueVSize();
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
+   const int tvsize = pfes->GetTrueVSize();
 
-   double * tvd = tv->GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv->Read();
+   Vector tvr; tvr.MakeRef(const_cast<Vector&>(*tv), 0, tvsize);
+   Vector tvi; tvi.MakeRef(const_cast<Vector&>(*tv), tvsize, tvsize);
 
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->Distribute(tvr);
    pgfi->Distribute(tvi);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexGridFunction::ParallelProject(Vector &tv) const
 {
-   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
-   HYPRE_Int size = pfes->GetTrueVSize();
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
+   const int tvsize = pfes->GetTrueVSize();
 
-   double * tvd = tv.GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv.Write();
+   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
+   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
 
+   pgfr->SyncMemory(*this);
+   pgfi->SyncMemory(*this);
    pgfr->ParallelProject(tvr);
    pgfi->ParallelProject(tvi);
+   pgfr->SyncAliasMemory(*this);
+   pgfi->SyncAliasMemory(*this);
+
+   tvr.SyncAliasMemory(tv);
+   tvi.SyncAliasMemory(tv);
 }
 
 
@@ -701,10 +815,16 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
    : Vector(2*(pfes->GetVSize())),
      conv(convention)
 {
-   plfr = new ParLinearForm(pfes, data);
-   plfi = new ParLinearForm(pfes, (data) ? &data[pfes->GetVSize()]:data);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
 
-   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   plfr = new ParLinearForm();
+   plfr->MakeRef(pfes, *this, 0);
+
+   plfi = new ParLinearForm();
+   plfi->MakeRef(pfes, *this, pfes->GetVSize());
+
+   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
 
    int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
    tdof_offsets = new HYPRE_Int[n+1];
@@ -724,12 +844,16 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
    : Vector(2*(pfes->GetVSize())),
      conv(convention)
 {
-   plfr = new ParLinearForm(pfes, plf_r);
-   plfr->SetData(data);
-   plfi = new ParLinearForm(pfes, plf_i);
-   plfi->SetData((data) ? &data[pfes->GetVSize()]:data);
+   UseDevice(true);
+   this->Vector::operator=(0.0);
 
-   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   plfr = new ParLinearForm(pfes, plf_r);
+   plfi = new ParLinearForm(pfes, plf_i);
+
+   plfr->MakeRef(pfes, *this, 0);
+   plfi->MakeRef(pfes, *this, pfes->GetVSize());
+
+   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
 
    int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
    tdof_offsets = new HYPRE_Int[n+1];
@@ -792,58 +916,71 @@ ParComplexLinearForm::AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
 void
 ParComplexLinearForm::Update(ParFiniteElementSpace *pf)
 {
-   ParFiniteElementSpace *pfes = (pf!=NULL)?pf:plfr->ParFESpace();
-   int vsize = pfes->GetVSize();
-   SetSize(2 * vsize);
+   ParFiniteElementSpace *pfes = (pf != NULL) ? pf : plfr->ParFESpace();
 
-   Vector vplfr(data, vsize);
-   Vector vplfi((data) ? &data[vsize] : data, vsize);
+   UseDevice(true);
+   SetSize(2 * pfes->GetVSize());
+   this->Vector::operator=(0.0);
 
-   plfr->Update(pfes, vplfr, 0);
-   plfi->Update(pfes, vplfi, 0);
+   plfr->MakeRef(pfes, *this, 0);
+   plfi->MakeRef(pfes, *this, pfes->GetVSize());
 }
 
 void
 ParComplexLinearForm::Assemble()
 {
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
    plfr->Assemble();
    plfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      *plfi *= -1.0;
-   }
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *plfi *= -1.0; }
+   plfr->SyncAliasMemory(*this);
+   plfi->SyncAliasMemory(*this);
 }
 
 void
 ParComplexLinearForm::ParallelAssemble(Vector &tv)
 {
-   HYPRE_Int size = plfr->ParFESpace()->GetTrueVSize();
+   const int tvsize = plfr->ParFESpace()->GetTrueVSize();
 
-   double * tvd = tv.GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv.Write();
+   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
+   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
 
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
+   plfr->SyncAliasMemory(*this);
+   plfi->SyncAliasMemory(*this);
+
+   tvr.SyncAliasMemory(tv);
+   tvi.SyncAliasMemory(tv);
 }
 
 HypreParVector *
 ParComplexLinearForm::ParallelAssemble()
 {
-   const ParFiniteElementSpace * pfes = plfr->ParFESpace();
+   const ParFiniteElementSpace *pfes = plfr->ParFESpace();
+   const int tvsize = pfes->GetTrueVSize();
 
-   HypreParVector * tv = new HypreParVector(pfes->GetComm(),
-                                            2*(pfes->GlobalTrueVSize()),
-                                            tdof_offsets);
+   HypreParVector *tv = new HypreParVector(pfes->GetComm(),
+                                           2*(pfes->GlobalTrueVSize()),
+                                           tdof_offsets);
 
-   HYPRE_Int size = pfes->GetTrueVSize();
-
-   double * tvd = tv->GetData();
-   Vector tvr(tvd, size);
-   Vector tvi((tvd) ? &tvd[size] : tvd, size);
+   tv->Write();
+   Vector tvr; tvr.MakeRef(*tv, 0, tvsize);
+   Vector tvi; tvi.MakeRef(*tv, tvsize, tvsize);
 
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
    plfr->ParallelAssemble(tvr);
    plfi->ParallelAssemble(tvi);
+   plfr->SyncAliasMemory(*this);
+   plfi->SyncAliasMemory(*this);
+
+   tvr.SyncAliasMemory(*tv);
+   tvi.SyncAliasMemory(*tv);
 
    return tv;
 }
@@ -851,13 +988,14 @@ ParComplexLinearForm::ParallelAssemble()
 complex<double>
 ParComplexLinearForm::operator()(const ParComplexGridFunction &gf) const
 {
-   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
+   plfr->SyncMemory(*this);
+   plfi->SyncMemory(*this);
+   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
    return complex<double>((*plfr)(gf.real()) - s * (*plfi)(gf.imag()),
                           (*plfr)(gf.imag()) + s * (*plfi)(gf.real()));
 }
 
 
-
 bool ParSesquilinearForm::RealInteg()
 {
    int nint = pblfr->GetFBFI()->Size() + pblfr->GetDBFI()->Size() +
@@ -964,7 +1102,6 @@ ParSesquilinearForm::ParallelAssemble()
    return new ComplexHypreParMatrix(pblfr->ParallelAssemble(),
                                     pblfi->ParallelAssemble(),
                                     true, true, conv);
-
 }
 
 void
@@ -974,35 +1111,45 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                       Vector &X, Vector &B,
                                       int ci)
 {
-   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
-   int vsize = pfes->GetVSize();
+   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
+   const int vsize = pfes->GetVSize();
 
-   // Allocate temporary vectors
-   Vector b_0(vsize); b_0 = 0.0;
+   // Allocate temporary vector
+   Vector b_0;
+   b_0.UseDevice(true);
+   b_0.SetSize(vsize);
+   b_0 = 0.0;
 
    // Extract the real and imaginary parts of the input vectors
    MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
+   x.Read();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
 
    MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   Vector b_r(b.GetData(), vsize);
-   Vector b_i(&(b.GetData())[vsize], vsize);
+   b.Read();
+   Vector b_r; b_r.MakeRef(b, 0, vsize);
+   Vector b_i; b_i.MakeRef(b, vsize, vsize);
 
    if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
 
-   int tvsize = pfes->GetTrueVSize();
-
+   const int tvsize = pfes->GetTrueVSize();
    OperatorHandle A_r, A_i;
 
+   X.UseDevice(true);
    X.SetSize(2 * tvsize);
-   B.SetSize(2 * tvsize);
+   X = 0.0;
 
-   Vector X_0(tvsize), B_0(tvsize);
-   Vector X_r(X.GetData(),tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
-   Vector B_r(B.GetData(), tvsize);
-   Vector B_i(&(B.GetData())[tvsize], tvsize);
+   B.UseDevice(true);
+   B.SetSize(2 * tvsize);
+   B = 0.0;
+
+   Vector X_r; X_r.MakeRef(X, 0, tvsize);
+   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
+   Vector B_r; B_r.MakeRef(B, 0, tvsize);
+   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
+
+   Vector X_0, B_0;
 
    if (RealInteg())
    {
@@ -1042,24 +1189,29 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    if (RealInteg() && ImagInteg())
    {
-      int n = ess_tdof_list.Size();
       // Modify RHS to conform with standard essential BC treatment
-      for (int k = 0; k < n; k++)
+      const int n = ess_tdof_list.Size();
+      auto d_B_r = B_r.Write();
+      auto d_B_i = B_i.Write();
+      auto d_X_r = X_r.Read();
+      auto d_X_i = X_i.Read();
+      auto d_idx = ess_tdof_list.Read();
+      MFEM_FORALL(i, n,
       {
-         int j=ess_tdof_list[k];
-         B_r(j) = X_r(j);
-         B_i(j) = X_i(j);
-      }
+         const int j = d_idx[i];
+         d_B_r[j] = d_X_r[j];
+         d_B_i[j] = d_X_i[j];
+      });
       // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      if ( A_i.Type() == Operator::Hypre_ParCSR )
+      if (A_i.Type() == Operator::Hypre_ParCSR)
       {
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
          for (int k = 0; k < n; k++)
          {
-            int j = ess_tdof_list[k];
+            const int j = ess_tdof_list[k];
             Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
       }
@@ -1076,6 +1228,16 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       b_i *= -1.0;
    }
 
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
+   b_r.SyncAliasMemory(b);
+   b_i.SyncAliasMemory(b);
+
+   X_r.SyncAliasMemory(X);
+   X_i.SyncAliasMemory(X);
+   B_r.SyncAliasMemory(B);
+   B_i.SyncAliasMemory(B);
+
    // A = A_r + i A_i
    A.Clear();
    if ( A_r.Type() == Operator::Hypre_ParCSR ||
@@ -1175,22 +1337,27 @@ void
 ParSesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                         Vector &x)
 {
-   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
+   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
 
    const Operator &P = *pfes->GetProlongationMatrix();
 
-   int vsize  = pfes->GetVSize();
-   int tvsize = X.Size() / 2;
+   const int vsize  = pfes->GetVSize();
+   const int tvsize = X.Size() / 2;
 
-   Vector X_r(X.GetData(), tvsize);
-   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   X.Read();
+   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
+   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
 
-   Vector x_r(x.GetData(), vsize);
-   Vector x_i(&(x.GetData())[vsize], vsize);
+   x.Write();
+   Vector x_r; x_r.MakeRef(x, 0, vsize);
+   Vector x_i; x_i.MakeRef(x, vsize, vsize);
 
    // Apply conforming prolongation
    P.Mult(X_r, x_r);
    P.Mult(X_i, x_i);
+
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
 }
 
 void

fem/complex_fem.hpp

@@ -99,8 +99,8 @@ public:
                      ComplexOperator::Convention
                      convention = ComplexOperator::HERMITIAN);
 
-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
-       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
+       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
 
        The pointer @a fes is not owned by the newly constructed object.
 
@@ -195,8 +195,8 @@ private:
    BilinearForm *blfr;
    BilinearForm *blfi;
 
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
+   /* These methods check if the real/imag parts of the sesquilinear form are
+      not empty */
    bool RealInteg();
    bool ImagInteg();
 
@@ -204,7 +204,7 @@ public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
        the same integrators as the BilinearForms @a bfr and @a bfi .
 
        The pointer @a fes is not owned by the newly constructed object.
@@ -324,7 +324,7 @@ protected:
 public:
 
    /* @brief Construct a ParComplexGridFunction associated with the
-      ParFiniteElementSpace @a *f. */
+      ParFiniteElementSpace @a *pf. */
    ParComplexGridFunction(ParFiniteElementSpace *pf);
 
    void Update();
@@ -416,8 +416,8 @@ public:
                         convention = ComplexOperator::HERMITIAN);
 
    /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plfr (real) and @a plfi
-       (imag) .
+       using the same integrators as the LinearForms @a plf_r (real) and
+       @a plf_i (imag).
 
       The pointer @a fes is not owned by the newly constructed object.
 

fem/eltrans.hpp

@@ -77,9 +77,6 @@ public:
 
    ElementTransformation();
 
-   /** @brief Force the reevaluation of the Jacobian in the next call. */
-   void Reset() { EvalState = 0; }
-
    /** @brief Set the integration point @a ip that weights and Jacobians will
        be evaluated at. */
    void SetIntPoint(const IntegrationPoint *ip)
@@ -360,17 +357,9 @@ private:
    // Evaluate the Hessian of the transformation at the IntPoint and store it
    // in d2Fdx2.
    virtual const DenseMatrix &EvalHessian();
-
 public:
-   IsoparametricTransformation() : FElem(NULL) {}
-
    /// Set the element that will be used to compute the transformations
-   void SetFE(const FiniteElement *FE)
-   {
-      MFEM_ASSERT(FE != NULL, "Must provide a valid FiniteElement object!");
-      EvalState = (FE != FElem) ? 0 : EvalState;
-      FElem = FE; geom = FE->GetGeomType();
-   }
+   void SetFE(const FiniteElement *FE) { FElem = FE; geom = FE->GetGeomType(); }
 
    /// Get the current element used to compute the transformations
    const FiniteElement* GetFE() const { return FElem; }
@@ -385,15 +374,12 @@ public:
        the column-vector of all basis functions evaluated at \f$ \hat x \f$ .
        The columns of @a P represent the control points in physical space
        defining the transformation. */
-   void SetPointMat(const DenseMatrix &pm) { PointMat = pm; EvalState = 0; }
+   void SetPointMat(const DenseMatrix &pm) { PointMat = pm; }
 
    /// Return the stored point matrix.
    const DenseMatrix &GetPointMat() const { return PointMat; }
 
-   /// @brief Write access to the stored point matrix. Use with caution.
-   /** If the point matrix is altered using this member function the Reset
-       function should also be called to force the reevaluation of the
-       Jacobian, etc.. */
+   /// Write access to the stored point matrix. Use with caution.
    DenseMatrix &GetPointMat() { return PointMat; }
 
    /// Set the FiniteElement Geometry for the reference elements being used.

fem/fe.cpp

@@ -7034,95 +7034,6 @@ void Poly_1D::Basis::Eval(const double y, Vector &u, Vector &d) const
    }
 }
 
-void Poly_1D::Basis::Eval(const double y, Vector &u, Vector &d,
-                          Vector &d2) const
-{
-   MFEM_VERIFY(etype == Barycentric,
-               "Basis::Eval with second order derivatives not implemented for"
-               " etype = " << etype);
-   switch (etype)
-   {
-      case ChangeOfBasis:
-      {
-         CalcBasis(Ai.Width() - 1, y, x, w);
-         Ai.Mult(x, u);
-         Ai.Mult(w, d);
-         // set d2 (not implemented yet)
-         break;
-      }
-      case Barycentric:
-      {
-         int i, k, p = x.Size() - 1;
-         double l, lp, lp2, lk, sk, si, sk2;
-
-         if (p == 0)
-         {
-            u(0) = 1.0;
-            d(0) = 0.0;
-            d2(0) = 0.0;
-            return;
-         }
-
-         lk = 1.0;
-         for (k = 0; k < p; k++)
-         {
-            if (y >= (x(k) + x(k+1))/2)
-            {
-               lk *= y - x(k);
-            }
-            else
-            {
-               for (i = k+1; i <= p; i++)
-               {
-                  lk *= y - x(i);
-               }
-               break;
-            }
-         }
-         l = lk * (y - x(k));
-
-         sk = 0.0;
-         sk2 = 0.0;
-         for (i = 0; i < k; i++)
-         {
-            si = 1.0/(y - x(i));
-            sk += si;
-            sk2 -= si * si;
-            u(i) = l * si * w(i);
-         }
-         u(k) = lk * w(k);
-         for (i++; i <= p; i++)
-         {
-            si = 1.0/(y - x(i));
-            sk += si;
-            sk2 -= si * si;
-            u(i) = l * si * w(i);
-         }
-         lp = l * sk + lk;
-         lp2 = lp * sk + l * sk2 + sk * lk;
-
-         for (i = 0; i < k; i++)
-         {
-            d(i) = (lp * w(i) - u(i))/(y - x(i));
-            d2(i) = (lp2 * w(i) - 2 * d(i))/(y - x(i));
-         }
-         d(k) = sk * u(k);
-         d2(k) = sk2 * u(k) + sk * d(k);
-         for (i++; i <= p; i++)
-         {
-            d(i) = (lp * w(i) - u(i))/(y - x(i));
-            d2(i) = (lp2 * w(i) - 2 * d(i))/(y - x(i));
-         }
-         break;
-      }
-      case Positive:
-         CalcBernstein(x.Size() - 1, y, u, d);
-         break;
-
-      default: break;
-   }
-}
-
 const int *Poly_1D::Binom(const int p)
 {
    if (binom.NumCols() <= p)
@@ -7678,7 +7589,6 @@ H1_SegmentElement::H1_SegmentElement(const int p, const int btype)
 #ifndef MFEM_THREAD_SAFE
    shape_x.SetSize(p+1);
    dshape_x.SetSize(p+1);
-   d2shape_x.SetSize(p+1);
 #endif
 
    Nodes.IntPoint(0).x = cp[0];
@@ -7727,25 +7637,6 @@ void H1_SegmentElement::CalcDShape(const IntegrationPoint &ip,
    }
 }
 
-void H1_SegmentElement::CalcHessian(const IntegrationPoint &ip,
-                                    DenseMatrix &Hessian) const
-{
-   const int p = order;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector shape_x(p+1), dshape_x(p+1), d2shape_x(p+1);
-#endif
-
-   basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);
-
-   Hessian(0,0) = d2shape_x(0);
-   Hessian(1,0) = d2shape_x(p);
-   for (int i = 1; i < p; i++)
-   {
-      Hessian(i+1,0) = d2shape_x(i);
-   }
-}
-
 void H1_SegmentElement::ProjectDelta(int vertex, Vector &dofs) const
 {
    const int p = order;
@@ -7786,8 +7677,6 @@ H1_QuadrilateralElement::H1_QuadrilateralElement(const int p, const int btype)
    shape_y.SetSize(p1);
    dshape_x.SetSize(p1);
    dshape_y.SetSize(p1);
-   d2shape_x.SetSize(p1);
-   d2shape_y.SetSize(p1);
 #endif
 
    int o = 0;
@@ -7841,30 +7730,6 @@ void H1_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
    }
 }
 
-void H1_QuadrilateralElement::CalcHessian(const IntegrationPoint &ip,
-                                          DenseMatrix &Hessian) const
-{
-   const int p = order;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector shape_x(p+1), shape_y(p+1), dshape_x(p+1), dshape_y(p+1),
-          d2shape_x(p+1), d2shape_y(p+1);
-#endif
-
-   basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);
-   basis1d.Eval(ip.y, shape_y, dshape_y, d2shape_y);
-
-   for (int o = 0, j = 0; j <= p; j++)
-   {
-      for (int i = 0; i <= p; i++)
-      {
-         Hessian(dof_map[o],0) = d2shape_x(i)*  shape_y(j);
-         Hessian(dof_map[o],1) =  dshape_x(i)* dshape_y(j);
-         Hessian(dof_map[o],2) =   shape_x(i)*d2shape_y(j);  o++;
-      }
-   }
-}
-
 void H1_QuadrilateralElement::ProjectDelta(int vertex, Vector &dofs) const
 {
    const int p = order;
@@ -7928,9 +7793,6 @@ H1_HexahedronElement::H1_HexahedronElement(const int p, const int btype)
    dshape_x.SetSize(p1);
    dshape_y.SetSize(p1);
    dshape_z.SetSize(p1);
-   d2shape_x.SetSize(p1);
-   d2shape_y.SetSize(p1);
-   d2shape_z.SetSize(p1);
 #endif
 
    int o = 0;
@@ -7987,35 +7849,6 @@ void H1_HexahedronElement::CalcDShape(const IntegrationPoint &ip,
          }
 }
 
-void H1_HexahedronElement::CalcHessian(const IntegrationPoint &ip,
-                                       DenseMatrix &Hessian) const
-{
-   const int p = order;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector shape_x(p+1),  shape_y(p+1),  shape_z(p+1);
-   Vector dshape_x(p+1), dshape_y(p+1), dshape_z(p+1);
-   Vector d2shape_x(p+1), d2shape_y(p+1), ds2hape_z(p+1);
-#endif
-
-   basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);
-   basis1d.Eval(ip.y, shape_y, dshape_y, d2shape_y);
-   basis1d.Eval(ip.z, shape_z, dshape_z, d2shape_z);
-
-   for (int o = 0, k = 0; k <= p; k++)
-      for (int j = 0; j <= p; j++)
-         for (int i = 0; i <= p; i++)
-         {
-            Hessian(dof_map[o],0) = d2shape_x(i)*  shape_y(j)*  shape_z(k);
-            Hessian(dof_map[o],1) =  dshape_x(i)* dshape_y(j)*  shape_z(k);
-            Hessian(dof_map[o],2) =  dshape_x(i)*  shape_y(j)* dshape_z(k);
-            Hessian(dof_map[o],3) =   shape_x(i)*d2shape_y(j)*  shape_z(k);
-            Hessian(dof_map[o],4) =   shape_x(i)* dshape_y(j)* dshape_z(k);
-            Hessian(dof_map[o],5) =   shape_x(i)*  shape_y(j)*d2shape_z(k);
-            o++;
-         }
-}
-
 void H1_HexahedronElement::ProjectDelta(int vertex, Vector &dofs) const
 {
    const int p = order;

fem/fe.hpp

@@ -446,7 +446,7 @@ public:
    /** Each row of the result DenseMatrix @a Hessian contains upper triangular
        part of the Hessian of one shape function.
        The order in 2D is {u_xx, u_xy, u_yy}.
-       The size (#dof x (#dim (#dim+1)/2) of @a Hessian must be set in advance.*/
+       The size (#dof x (#dim (#dim-1)/2) of @a Hessian must be set in advance.*/
    virtual void CalcHessian (const IntegrationPoint &ip,
                              DenseMatrix &Hessian) const;
 
@@ -1850,7 +1850,6 @@ public:
       Basis(const int p, const double *nodes, EvalType etype = Barycentric);
       void Eval(const double x, Vector &u) const;
       void Eval(const double x, Vector &u, Vector &d) const;
-      void Eval(const double x, Vector &u, Vector &d, Vector &d2) const;
    };
 
 private:
@@ -2101,7 +2100,7 @@ class H1_SegmentElement : public NodalTensorFiniteElement
 {
 private:
 #ifndef MFEM_THREAD_SAFE
-   mutable Vector shape_x, dshape_x, d2shape_x;
+   mutable Vector shape_x, dshape_x;
 #endif
 
 public:
@@ -2110,8 +2109,6 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
-   virtual void CalcHessian(const IntegrationPoint &ip,
-                            DenseMatrix &Hessian) const;
    virtual void ProjectDelta(int vertex, Vector &dofs) const;
 };
 
@@ -2121,7 +2118,7 @@ class H1_QuadrilateralElement : public NodalTensorFiniteElement
 {
 private:
 #ifndef MFEM_THREAD_SAFE
-   mutable Vector shape_x, shape_y, dshape_x, dshape_y, d2shape_x, d2shape_y;
+   mutable Vector shape_x, shape_y, dshape_x, dshape_y;
 #endif
 
 public:
@@ -2131,8 +2128,6 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
-   virtual void CalcHessian(const IntegrationPoint &ip,
-                            DenseMatrix &Hessian) const;
    virtual void ProjectDelta(int vertex, Vector &dofs) const;
 };
 
@@ -2142,8 +2137,7 @@ class H1_HexahedronElement : public NodalTensorFiniteElement
 {
 private:
 #ifndef MFEM_THREAD_SAFE
-   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z,
-           d2shape_x, d2shape_y, d2shape_z;
+   mutable Vector shape_x, shape_y, shape_z, dshape_x, dshape_y, dshape_z;
 #endif
 
 public:
@@ -2152,8 +2146,6 @@ public:
    virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
    virtual void CalcDShape(const IntegrationPoint &ip,
                            DenseMatrix &dshape) const;
-   virtual void CalcHessian(const IntegrationPoint &ip,
-                            DenseMatrix &Hessian) const;
    virtual void ProjectDelta(int vertex, Vector &dofs) const;
 };
 

fem/gridfunc.cpp

@@ -199,8 +199,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
    if (f != fes) { Destroy(); }
    fes = f;
    v.UseDevice(true);
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
-                    fes->GetVSize(), true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    sequence = fes->GetSequence();
 }
 

fem/libceed/ceed.cpp

@@ -415,59 +415,6 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedVectorCreate(ceed, fes.GetNDofs(), &ceedData.v);
 }
 
-void CeedAddMultPA(const CeedData *ceedDataPtr,
-                   const Vector &x,
-                   Vector &y)
-{
-   const CeedScalar *x_ptr;
-   CeedScalar *y_ptr;
-   CeedMemType mem;
-   CeedGetPreferredMemType(internal::ceed, &mem);
-   if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
-   {
-      x_ptr = x.Read();
-      y_ptr = y.ReadWrite();
-   }
-   else
-   {
-      x_ptr = x.HostRead();
-      y_ptr = y.HostReadWrite();
-      mem = CEED_MEM_HOST;
-   }
-   CeedVectorSetArray(ceedDataPtr->u, mem, CEED_USE_POINTER,
-                      const_cast<CeedScalar*>(x_ptr));
-   CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, y_ptr);
-
-   CeedOperatorApplyAdd(ceedDataPtr->oper, ceedDataPtr->u, ceedDataPtr->v,
-                        CEED_REQUEST_IMMEDIATE);
-
-   CeedVectorTakeArray(ceedDataPtr->u, mem, const_cast<CeedScalar**>(&x_ptr));
-   CeedVectorTakeArray(ceedDataPtr->v, mem, &y_ptr);
-}
-
-void CeedAssembleDiagonalPA(const CeedData *ceedDataPtr,
-                            Vector &diag)
-{
-   CeedScalar *d_ptr;
-   CeedMemType mem;
-   CeedGetPreferredMemType(internal::ceed, &mem);
-   if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
-   {
-      d_ptr = diag.ReadWrite();
-   }
-   else
-   {
-      d_ptr = diag.HostReadWrite();
-      mem = CEED_MEM_HOST;
-   }
-   CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, d_ptr);
-
-   CeedOperatorLinearAssembleAddDiagonal(ceedDataPtr->oper, ceedDataPtr->v,
-                                         CEED_REQUEST_IMMEDIATE);
-
-   CeedVectorTakeArray(ceedDataPtr->v, mem, &d_ptr);
-}
-
 } // namespace mfem
 
 #endif // MFEM_USE_CEED

fem/libceed/ceed.hpp

@@ -16,7 +16,6 @@
 
 #ifdef MFEM_USE_CEED
 #include "../../general/device.hpp"
-#include "../../linalg/vector.hpp"
 #include <ceed.h>
 
 namespace mfem
@@ -145,15 +144,6 @@ const std::string &GetCeedPath();
 void CeedPAAssemble(const CeedPAOperator& op,
                     CeedData& ceedData);
 
-/** @brief Function that applies a libCEED PA operator. */
-void CeedAddMultPA(const CeedData *ceedDataPtr,
-                   const Vector &x,
-                   Vector &y);
-
-/** @brief Function that assembles a libCEED PA operator diagonal. */
-void CeedAssembleDiagonalPA(const CeedData *ceedDataPtr,
-                            Vector &diag);
-
 /** @brief Function that determines if a CEED kernel should be used, based on
     the current mfem::Device configuration. */
 inline bool DeviceCanUseCeed()

fem/linearform.cpp

@@ -204,6 +204,14 @@ void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
    ResetDeltaLocations();
 }
 
+void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
+{
+   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
+   fes = f;
+   v.UseDevice(true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+}
+
 void LinearForm::AssembleDelta()
 {
    if (dlfi_delta.Size() == 0) { return; }

fem/linearform.hpp

@@ -26,7 +26,7 @@ protected:
    /// FE space on which the LinearForm lives. Not owned.
    FiniteElementSpace *fes;
 
-   /** @brief Indicates the LinerFormIntegrator%s stored in #dlfi, #dlfi_delta,
+   /** @brief Indicates the LinearFormIntegrator%s stored in #dlfi, #dlfi_delta,
        #blfi, and #flfi are owned by another LinearForm. */
    int extern_lfs;
 
@@ -175,6 +175,15 @@ public:
        @note This method does not perform assembly. */
    void Update(FiniteElementSpace *f, Vector &v, int v_offset);
 
+   /** @brief Make the LinearForm reference external data on a new
+       FiniteElementSpace. */
+   /** This method changes the FiniteElementSpace associated with the LinearForm
+       @a *f and sets the data of the Vector @a v (plus the @a v_offset)
+       as external data in the LinearForm.
+       @note This version of the method will also perform bounds checks when
+       the build option MFEM_DEBUG is enabled. */
+   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
+
    /// Return the action of the LinearForm as a linear mapping.
    /** Linear forms are linear functionals which map GridFunctions to
        the real numbers.  This method performs this mapping which in

fem/plinearform.cpp

@@ -21,7 +21,6 @@ namespace mfem
 void ParLinearForm::Update(ParFiniteElementSpace *pf)
 {
    if (pf) { pfes = pf; }
-
    LinearForm::Update(pfes);
 }
 
@@ -31,6 +30,18 @@ void ParLinearForm::Update(ParFiniteElementSpace *pf, Vector &v, int v_offset)
    LinearForm::Update(pf,v,v_offset);
 }
 
+void ParLinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
+{
+   LinearForm::MakeRef(f, v, v_offset);
+   pfes = dynamic_cast<ParFiniteElementSpace*>(f);
+}
+
+void ParLinearForm::MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset)
+{
+   LinearForm::MakeRef(pf, v, v_offset);
+   pfes = pf;
+}
+
 void ParLinearForm::ParallelAssemble(Vector &tv)
 {
    const Operator* prolong = pfes->GetProlongationMatrix();

fem/plinearform.hpp

@@ -92,6 +92,25 @@ public:
        @note This method does not perform assembly. */
    void Update(ParFiniteElementSpace *pf, Vector &v, int v_offset);
 
+
+   /** @brief Make the ParLinearForm reference external data on a new
+       FiniteElementSpace. */
+   /** This method changes the FiniteElementSpace associated with the ParLinearForm
+       to @a *f and sets the data of the Vector @a v (plus the @a v_offset) as external
+       data in the ParLinearForm.
+       @note This version of the method will also perform bounds checks when
+       the build option MFEM_DEBUG is enabled. */
+   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
+
+   /** @brief Make the ParLinearForm reference external data on a new
+       ParFiniteElementSpace. */
+   /** This method changes the ParFiniteElementSpace associated with the ParLinearForm
+       to @a *pf and sets the data of the Vector @a v (plus the @a v_offset) as external
+       data in the ParLinearForm.
+       @note This version of the method will also perform bounds checks when
+       the build option MFEM_DEBUG is enabled. */
+   void MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset);
+
    /// Assemble the vector on the true dofs, i.e. P^t v.
    void ParallelAssemble(Vector &tv);
 

fem/quadinterpolator.cpp

@@ -62,7 +62,6 @@ void QuadratureInterpolator::Eval2D(
    const int nq = maps.nqpt;
    const int ND = T_ND ? T_ND : nd;
    const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
    MFEM_VERIFY(ND <= MAX_ND2D, "");
    MFEM_VERIFY(NQ <= MAX_NQ2D, "");
@@ -73,24 +72,22 @@ void QuadratureInterpolator::Eval2D(
    auto val = Reshape(q_val.Write(), NQ, VDIM, NE);
    auto der = Reshape(q_der.Write(), NQ, VDIM, 2, NE);
    auto det = Reshape(q_det.Write(), NQ, NE);
-   MFEM_FORALL_2D(e, NE, NMAX, 1, 1,
+   MFEM_FORALL(e, NE,
    {
       const int ND = T_ND ? T_ND : nd;
       const int NQ = T_NQ ? T_NQ : nq;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
       constexpr int max_ND = T_ND ? T_ND : MAX_ND2D;
       constexpr int max_VDIM = T_VDIM ? T_VDIM : MAX_VDIM2D;
-      MFEM_SHARED double s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
+      double s_E[max_VDIM*max_ND];
+      for (int d = 0; d < ND; d++)
       {
          for (int c = 0; c < VDIM; c++)
          {
             s_E[c+d*VDIM] = E(d,c,e);
          }
       }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
+      for (int q = 0; q < NQ; ++q)
       {
          if (eval_flags & VALUES)
          {
@@ -153,7 +150,6 @@ void QuadratureInterpolator::Eval3D(
    const int nq = maps.nqpt;
    const int ND = T_ND ? T_ND : nd;
    const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
    MFEM_VERIFY(ND <= MAX_ND3D, "");
    MFEM_VERIFY(NQ <= MAX_NQ3D, "");
@@ -164,24 +160,22 @@ void QuadratureInterpolator::Eval3D(
    auto val = Reshape(q_val.Write(), NQ, VDIM, NE);
    auto der = Reshape(q_der.Write(), NQ, VDIM, 3, NE);
    auto det = Reshape(q_det.Write(), NQ, NE);
-   MFEM_FORALL_2D(e, NE, NMAX, 1, 1,
+   MFEM_FORALL(e, NE,
    {
       const int ND = T_ND ? T_ND : nd;
       const int NQ = T_NQ ? T_NQ : nq;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
       constexpr int max_ND = T_ND ? T_ND : MAX_ND3D;
       constexpr int max_VDIM = T_VDIM ? T_VDIM : MAX_VDIM3D;
-      MFEM_SHARED double s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
+      double s_E[max_VDIM*max_ND];
+      for (int d = 0; d < ND; d++)
       {
          for (int c = 0; c < VDIM; c++)
          {
             s_E[c+d*VDIM] = E(d,c,e);
          }
       }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
+      for (int q = 0; q < NQ; ++q)
       {
          if (eval_flags & VALUES)
          {

fem/tmop.cpp

@@ -1968,11 +1968,11 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
    Jpt.SetSize(dim);
    PMatI.UseExternalData(elfun.GetData(), dof, dim);
 
-   const IntegrationRule &ir = EnergyIntegrationRule(el);
+   const IntegrationRule *ir = EnergyIntegrationRule(el);
 
    energy = 0.0;
-   DenseTensor Jtr(dim, dim, ir.GetNPoints());
-   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+   DenseTensor Jtr(dim, dim, ir->GetNPoints());
+   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
 
    // Limited case.
    Vector shape, p, p0, d_vals;
@@ -1989,11 +1989,11 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
       nodes0->GetSubVector(pos_dofs, pos0V);
       if (lim_dist)
       {
-         lim_dist->GetValues(T.ElementNo, ir, d_vals);
+         lim_dist->GetValues(T.ElementNo, *ir, d_vals);
       }
       else
       {
-         d_vals.SetSize(ir.GetNPoints()); d_vals = 1.0;
+         d_vals.SetSize(ir->GetNPoints()); d_vals = 1.0;
       }
    }
 
@@ -2019,13 +2019,13 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
    Vector zeta_q, zeta0_q;
    if (adaptive_limiting)
    {
-      zeta->GetValues(T.ElementNo, ir, zeta_q);
-      zeta_0->GetValues(T.ElementNo, ir, zeta0_q);
+      zeta->GetValues(T.ElementNo, *ir, zeta_q);
+      zeta_0->GetValues(T.ElementNo, *ir, zeta0_q);
    }
 
-   for (int i = 0; i < ir.GetNPoints(); i++)
+   for (int i = 0; i < ir->GetNPoints(); i++)
    {
-      const IntegrationPoint &ip = ir.IntPoint(i);
+      const IntegrationPoint &ip = ir->IntPoint(i);
       const DenseMatrix &Jtr_i = Jtr(i);
       metric->SetTargetJacobian(Jtr_i);
       CalcInverse(Jtr_i, Jrt);
@@ -2105,14 +2105,14 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
    elvect.SetSize(dof*dim);
    PMatO.UseExternalData(elvect.GetData(), dof, dim);
 
-   const IntegrationRule &ir = ActionIntegrationRule(el);
-   const int nqp = ir.GetNPoints();
+   const IntegrationRule *ir = ActionIntegrationRule(el);
+   const int nqp = ir->GetNPoints();
 
    elvect = 0.0;
    Vector weights(nqp);
    DenseTensor Jtr(dim, dim, nqp);
    DenseTensor dJtr(dim, dim, dim*nqp);
-   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
 
    // Limited case.
    DenseMatrix pos0;
@@ -2129,7 +2129,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       nodes0->GetSubVector(pos_dofs, pos0V);
       if (lim_dist)
       {
-         lim_dist->GetValues(T.ElementNo, ir, d_vals);
+         lim_dist->GetValues(T.ElementNo, *ir, d_vals);
       }
       else
       {
@@ -2144,12 +2144,11 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       Tpr = new IsoparametricTransformation;
       Tpr->SetFE(&el);
       Tpr->ElementNo = T.ElementNo;
-      Tpr->ElementType = ElementTransformation::ELEMENT;
       Tpr->Attribute = T.Attribute;
       Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
       if (exact_action)
       {
-         targetC->ComputeElementTargetsGradient(ir, elfun, *Tpr, dJtr);
+         targetC->ComputeElementTargetsGradient(*ir, elfun, *Tpr, dJtr);
       }
    }
 
@@ -2159,7 +2158,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
 
    for (int q = 0; q < nqp; q++)
    {
-      const IntegrationPoint &ip = ir.IntPoint(q);
+      const IntegrationPoint &ip = ir->IntPoint(q);
       const DenseMatrix &Jtr_q = Jtr(q);
       metric->SetTargetJacobian(Jtr_q);
       CalcInverse(Jtr_q, Jrt);
@@ -2186,7 +2185,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
          DenseMatrix dwdx(dim);
          for (int d = 0; d < dim; d++)
          {
-            const DenseMatrix &dJtr_q = dJtr(q + d * nqp);
+            const DenseMatrix &dJtr_q = dJtr(q + d*ir->GetNPoints());
             Mult(Jrt, dJtr_q, dwdx );
             d_detW_dx(d) = dwdx.Trace();
          }
@@ -2221,7 +2220,7 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       }
    }
 
-   if (zeta) { AssembleElemVecAdaptLim(el, weights, *Tpr, ir, PMatO); }
+   if (zeta) { AssembleElemVecAdaptLim(el, weights, *Tpr, *ir, PMatO); }
 
    delete Tpr;
 }
@@ -2240,13 +2239,13 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
    PMatI.UseExternalData(elfun.GetData(), dof, dim);
    elmat.SetSize(dof*dim);
 
-   const IntegrationRule &ir = GradientIntegrationRule(el);
-   const int nqp = ir.GetNPoints();
+   const IntegrationRule *ir = GradientIntegrationRule(el);
+   const int nqp = ir->GetNPoints();
 
    elmat = 0.0;
    Vector weights(nqp);
    DenseTensor Jtr(dim, dim, nqp);
-   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
 
    // Limited case.
    DenseMatrix pos0, grad_grad;
@@ -2263,7 +2262,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
       nodes0->GetSubVector(pos_dofs, pos0V);
       if (lim_dist)
       {
-         lim_dist->GetValues(T.ElementNo, ir, d_vals);
+         lim_dist->GetValues(T.ElementNo, *ir, d_vals);
       }
       else
       {
@@ -2285,7 +2284,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
 
    for (int q = 0; q < nqp; q++)
    {
-      const IntegrationPoint &ip = ir.IntPoint(q);
+      const IntegrationPoint &ip = ir->IntPoint(q);
       const DenseMatrix &Jtr_q = Jtr(q);
       metric->SetTargetJacobian(Jtr_q);
       CalcInverse(Jtr_q, Jrt);
@@ -2302,6 +2301,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
 
       // TODO: derivatives of adaptivity-based targets.
 
+      // TODO optimize by symmetry.
       if (coeff0)
       {
          el.CalcShape(ip, shape);
@@ -2327,7 +2327,7 @@ void TMOP_Integrator::AssembleElementGradExact(const FiniteElement &el,
       }
    }
 
-   if (zeta) { AssembleElemGradAdaptLim(el, weights, *Tpr, ir, elmat); }
+   if (zeta) { AssembleElemGradAdaptLim(el, weights, *Tpr, *ir, elmat); }
 
    delete Tpr;
 }
@@ -2498,10 +2498,10 @@ void TMOP_Integrator::AssembleElementVectorFD(const FiniteElement &el,
    // Contributions from adaptive limiting (exact derivatives).
    if (zeta)
    {
-      const IntegrationRule &ir = ActionIntegrationRule(el);
-      const int nqp = ir.GetNPoints();
+      const IntegrationRule *ir = ActionIntegrationRule(el);
+      const int nqp = ir->GetNPoints();
       DenseTensor Jtr(dim, dim, nqp);
-      targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+      targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
 
       IsoparametricTransformation Tpr;
       Tpr.SetFE(&el);
@@ -2513,11 +2513,11 @@ void TMOP_Integrator::AssembleElementVectorFD(const FiniteElement &el,
       Vector weights(nqp);
       for (int q = 0; q < nqp; q++)
       {
-         weights(q) = ir.IntPoint(q).weight * Jtr(q).Det();
+         weights(q) = ir->IntPoint(q).weight * Jtr(q).Det();
       }
 
       PMatO.UseExternalData(elvect.GetData(), dof, dim);
-      AssembleElemVecAdaptLim(el, weights, Tpr, ir, PMatO);
+      AssembleElemVecAdaptLim(el, weights, Tpr, *ir, PMatO);
    }
 }
 
@@ -2594,10 +2594,10 @@ void TMOP_Integrator::AssembleElementGradFD(const FiniteElement &el,
    // Contributions from adaptive limiting.
    if (zeta)
    {
-      const IntegrationRule &ir = GradientIntegrationRule(el);
-      const int nqp = ir.GetNPoints();
+      const IntegrationRule *ir = GradientIntegrationRule(el);
+      const int nqp = ir->GetNPoints();
       DenseTensor Jtr(dim, dim, nqp);
-      targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+      targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);
 
       IsoparametricTransformation Tpr;
       Tpr.SetFE(&el);
@@ -2609,10 +2609,10 @@ void TMOP_Integrator::AssembleElementGradFD(const FiniteElement &el,
       Vector weights(nqp);
       for (int q = 0; q < nqp; q++)
       {
-         weights(q) = ir.IntPoint(q).weight * Jtr(q).Det();
+         weights(q) = ir->IntPoint(q).weight * Jtr(q).Det();
       }
 
-      AssembleElemGradAdaptLim(el, weights, Tpr, ir, elmat);
+      AssembleElemGradAdaptLim(el, weights, Tpr, *ir, elmat);
    }
 }
 
@@ -2642,32 +2642,33 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
    Array<int> vdofs;
    Vector x_vals;
    const FiniteElementSpace* const fes = x.FESpace();
+   const FiniteElement *fe = fes->GetFE(0);
 
-   const int dim = fes->GetMesh()->Dimension();
+   const int dof = fes->GetFE(0)->GetDof(), dim = fes->GetFE(0)->GetDim();
+
+   DSh.SetSize(dof, dim);
    Jrt.SetSize(dim);
    Jpr.SetSize(dim);
    Jpt.SetSize(dim);
 
+   const IntegrationRule *ir = EnergyIntegrationRule(*fe);
+   const int nqp =  ir->GetNPoints();
+   DenseTensor Jtr(dim, dim, nqp);
+
    metric_energy = 0.0;
    lim_energy = 0.0;
    for (int i = 0; i < fes->GetNE(); i++)
    {
-      const FiniteElement *fe = fes->GetFE(i);
-      const IntegrationRule &ir = EnergyIntegrationRule(*fe);
-      const int nqp = ir.GetNPoints();
-      DenseTensor Jtr(dim, dim, nqp);
-      const int dof = fe->GetDof();
-      DSh.SetSize(dof, dim);
-
+      fe = fes->GetFE(i);
       fes->GetElementVDofs(i, vdofs);
       x.GetSubVector(vdofs, x_vals);
       PMatI.UseExternalData(x_vals.GetData(), dof, dim);
 
-      targetC->ComputeElementTargets(i, *fe, ir, x_vals, Jtr);
+      targetC->ComputeElementTargets(i, *fe, *ir, x_vals, Jtr);
 
       for (int q = 0; q < nqp; q++)
       {
-         const IntegrationPoint &ip = ir.IntPoint(q);
+         const IntegrationPoint &ip = ir->IntPoint(q);
          metric->SetTargetJacobian(Jtr(q));
          CalcInverse(Jtr(q), Jrt);
          const double weight = ip.weight * Jtr(q).Det();
@@ -2691,9 +2692,9 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
                                     const FiniteElementSpace &fes)
 {
    const FiniteElement *fe = fes.GetFE(0);
-   const IntegrationRule &ir = EnergyIntegrationRule(*fe);
+   const IntegrationRule *ir = EnergyIntegrationRule(*fe);
    const int NE = fes.GetMesh()->GetNE(), dim = fe->GetDim(),
-             dof = fe->GetDof(), nsp = ir.GetNPoints();
+             dof = fe->GetDof(), nsp = ir->GetNPoints();
 
    Array<int> xdofs(dof * dim);
    DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
@@ -2710,7 +2711,7 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
       detv_sum = 0.;
       for (int j = 0; j < nsp; j++)
       {
-         fes.GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+         fes.GetFE(i)->CalcDShape(ir->IntPoint(j), dshape);
          MultAtB(pos, dshape, Jpr);
          detv_sum += std::fabs(Jpr.Det());
       }

fem/tmop.hpp

@@ -890,10 +890,6 @@ protected:
    TMOP_QualityMetric *metric;        // not owned
    const TargetConstructor *targetC;  // not owned
 
-   // Custom integration rules.
-   IntegrationRules *IntegRules;
-   int integ_order;
-
    // Weight Coefficient multiplying the quality metric term.
    Coefficient *coeff1; // not owned, if NULL -> coeff1 is 1.
    // Normalization factor for the metric term.
@@ -992,21 +988,17 @@ protected:
       nodes0 = NULL; coeff0 = NULL; lim_dist = NULL; lim_func = NULL;
    }
 
-   const IntegrationRule &EnergyIntegrationRule(const FiniteElement &el) const
+   const IntegrationRule *EnergyIntegrationRule(const FiniteElement &el) const
    {
-      if (IntegRules)
-      {
-         return IntegRules->Get(el.GetGeomType(), integ_order);
-      }
-      return (IntRule) ? *IntRule
-             /*     */ : IntRules.Get(el.GetGeomType(), 2*el.GetOrder() + 3);
+      return (IntRule) ? IntRule
+             /*     */ : &(IntRules.Get(el.GetGeomType(), 2*el.GetOrder() + 3));
    }
-   const IntegrationRule &ActionIntegrationRule(const FiniteElement &el) const
+   const IntegrationRule *ActionIntegrationRule(const FiniteElement &el) const
    {
       // TODO the energy most likely needs less integration points.
       return EnergyIntegrationRule(el);
    }
-   const IntegrationRule &GradientIntegrationRule(const FiniteElement &el) const
+   const IntegrationRule *GradientIntegrationRule(const FiniteElement &el) const
    {
       // TODO the action and energy most likely need less integration points.
       return EnergyIntegrationRule(el);
@@ -1016,7 +1008,7 @@ public:
    /** @param[in] m  TMOP_QualityMetric that will be integrated (not owned).
        @param[in] tc Target-matrix construction algorithm to use (not owned). */
    TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
-      : metric(m), targetC(tc), IntegRules(NULL), integ_order(-1),
+      : metric(m), targetC(tc),
         coeff1(NULL), metric_normal(1.0),
         nodes0(NULL), coeff0(NULL),
         lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
@@ -1027,14 +1019,6 @@ public:
 
    ~TMOP_Integrator();
 
-   /// Prescribe a set of integration rules; relevant for mixed meshes.
-   /** This function has priority over SetIntRule(), if both are called. */
-   void SetIntegrationRules(IntegrationRules &irules, int order)
-   {
-      IntegRules = &irules;
-      integ_order = order;
-   }
-
    /// Sets a scaling Coefficient for the quality metric term of the integrator.
    /** With this addition, the integrator becomes
           @f$ \int w1 W(Jpt) dx @f$.

fem/tmop_tools.cpp

@@ -176,8 +176,8 @@ SerialAdvectorCGOper::SerialAdvectorCGOper(const Vector &x_start,
 
    MassIntegrator *Minteg = new MassIntegrator;
    M.AddDomainIntegrator(Minteg);
-   M.Assemble(0);
-   M.Finalize(0);
+   M.Assemble();
+   M.Finalize();
 }
 
 void SerialAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
@@ -220,8 +220,8 @@ ParAdvectorCGOper::ParAdvectorCGOper(const Vector &x_start,
 
    MassIntegrator *Minteg = new MassIntegrator;
    M.AddDomainIntegrator(Minteg);
-   M.Assemble(0);
-   M.Finalize(0);
+   M.Assemble();
+   M.Finalize();
 }
 
 void ParAdvectorCGOper::Mult(const Vector &ind, Vector &di_dt) const
@@ -353,12 +353,13 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
       energy_in = nlf->GetEnergy(x);
    }
 
-   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetMesh()->Dimension();
-   Array<int> xdofs;
-   DenseMatrix Jpr(dim);
-
-   // Get the local prolongation of the solution vector.
+   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
+             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
+   Array<int> xdofs(dof * dim);
+   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
+   Vector posV(pos.Data(), dof * dim);
    Vector x_out_loc(fes->GetVSize());
+
    if (serial)
    {
       const SparseMatrix *cP = fes->GetConformingProlongation();
@@ -372,23 +373,15 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
    }
 #endif
 
-   // Check if the starting mesh (given by x) is inverted.
-   // Note that x hasn't been modified by the Newton update yet.
    double min_detJ = infinity();
    for (int i = 0; i < NE; i++)
    {
-      const int dof = fes->GetFE(i)->GetDof();
-      DenseMatrix dshape(dof, dim), pos(dof, dim);
-      Vector posV(pos.Data(), dof * dim);
-
       fes->GetElementVDofs(i, xdofs);
       x_out_loc.GetSubVector(xdofs, posV);
 
-      const IntegrationRule &irule = GetIntegrationRule(*fes->GetFE(i));
-      const int nsp = irule.GetNPoints();
       for (int j = 0; j < nsp; j++)
       {
-         fes->GetFE(i)->CalcDShape(irule.IntPoint(j), dshape);
+         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
          MultAtB(pos, dshape, Jpr);
          min_detJ = std::min(min_detJ, Jpr.Det());
       }
@@ -401,18 +394,18 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
                     p_nlf->ParFESpace()->GetComm());
    }
 #endif
-   const bool untangling = (min_detJ_all <= 0) ? true : false;
+   bool untangling = false;
+   if (min_detJ_all <= 0) { untangling = true; }
 
    const bool have_b = (b.Size() == Height());
 
    Vector x_out(x.Size());
    bool x_out_ok = false;
    double scale = 1.0, energy_out = 0.0;
-   const double norm0 = Norm(r);
+   double norm0 = Norm(r);
 
    const double detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
 
-   // Perform the line search.
    for (int i = 0; i < 12; i++)
    {
       add(x, -scale, c, x_out);
@@ -436,18 +429,11 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
          int jac_ok = 1;
          for (int i = 0; i < NE; i++)
          {
-            const int dof = fes->GetFE(i)->GetDof();
-            DenseMatrix dshape(dof, dim), pos(dof, dim);
-            Vector posV(pos.Data(), dof * dim);
-
             fes->GetElementVDofs(i, xdofs);
             x_out_loc.GetSubVector(xdofs, posV);
-
-            const IntegrationRule &irule = GetIntegrationRule(*fes->GetFE(i));
-            const int nsp = irule.GetNPoints();
             for (int j = 0; j < nsp; j++)
             {
-               fes->GetFE(i)->CalcDShape(irule.IntPoint(j), dshape);
+               fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
                MultAtB(pos, dshape, Jpr);
                if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
             }

fem/tmop_tools.hpp

@@ -118,39 +118,16 @@ protected:
 
    // Quadrature points that are checked for negative Jacobians etc.
    const IntegrationRule &ir;
-   // These fields are relevant for mixed meshes.
-   IntegrationRules *IntegRules;
-   int integ_order;
-
-   const IntegrationRule &GetIntegrationRule(const FiniteElement &el) const
-   {
-      if (IntegRules)
-      {
-         return IntegRules->Get(el.GetGeomType(), integ_order);
-      }
-      return ir;
-   }
 
    void UpdateDiscreteTC(const TMOP_Integrator &ti, const Vector &x_new) const;
 
 public:
 #ifdef MFEM_USE_MPI
    TMOPNewtonSolver(MPI_Comm comm, const IntegrationRule &irule, int type = 0)
-      : LBFGSSolver(comm), solver_type(type), parallel(true),
-        ir(irule), IntegRules(NULL), integ_order(-1) { }
+      : LBFGSSolver(comm), solver_type(type), parallel(true), ir(irule) { }
 #endif
    TMOPNewtonSolver(const IntegrationRule &irule, int type = 0)
-      : LBFGSSolver(), solver_type(type), parallel(false),
-        ir(irule), IntegRules(NULL), integ_order(-1) { }
-
-   /// Prescribe a set of integration rules; relevant for mixed meshes.
-   /** If called, this function has priority over the IntegrationRule given to
-       the constructor of the class. */
-   void SetIntegrationRules(IntegrationRules &irules, int order)
-   {
-      IntegRules = &irules;
-      integ_order = order;
-   }
+      : LBFGSSolver(), solver_type(type), parallel(false), ir(irule) { }
 
    virtual double ComputeScalingFactor(const Vector &x, const Vector &b) const;
 

linalg/complex_operator.cpp

@@ -26,10 +26,10 @@ ComplexOperator::ComplexOperator(Operator * Op_Real, Operator * Op_Imag,
    , ownReal_(ownReal)
    , ownImag_(ownImag)
    , convention_(convention)
-   , x_r_(NULL, width / 2)
-   , x_i_(NULL, width / 2)
-   , y_r_(NULL, height / 2)
-   , y_i_(NULL, height / 2)
+   , x_r_()
+   , x_i_()
+   , y_r_()
+   , y_i_()
    , u_(NULL)
    , v_(NULL)
 {}
@@ -68,14 +68,26 @@ const Operator & ComplexOperator::imag() const
 
 void ComplexOperator::Mult(const Vector &x, Vector &y) const
 {
-   double * x_data = x.GetData();
-   x_r_.SetData(x_data);
-   x_i_.SetData(&x_data[width / 2]);
+   x.Read();
+   y.UseDevice(true); y = 0.0;
 
-   y_r_.SetData(&y[0]);
-   y_i_.SetData(&y[height / 2]);
+   x_r_.MakeRef(const_cast<Vector&>(x), 0, width/2);
+   x_i_.MakeRef(const_cast<Vector&>(x), width/2, width/2);
+
+   y_r_.MakeRef(y, 0, height/2);
+   y_i_.MakeRef(y, height/2, height/2);
 
    this->Mult(x_r_, x_i_, y_r_, y_i_);
+
+   y_r_.SyncAliasMemory(y);
+   y_i_.SyncAliasMemory(y);
+
+   // Destroy alias vectors to prevent dangling aliases when the base vectors
+   // are deleted
+   x_r_.Destroy();
+   x_i_.Destroy();
+   y_r_.Destroy();
+   y_i_.Destroy();
 }
 
 void ComplexOperator::Mult(const Vector &x_r, const Vector &x_i,
@@ -91,31 +103,47 @@ void ComplexOperator::Mult(const Vector &x_r, const Vector &x_i,
       y_r = 0.0;
       y_i = 0.0;
    }
+
    if (Op_Imag_)
    {
-      if (!v_) { v_ = new Vector(Op_Imag_->Height()); }
+      if (!v_) { v_ = new Vector(); }
+      v_->UseDevice(true);
+      v_->SetSize(Op_Imag_->Height());
+
       Op_Imag_->Mult(x_i, *v_);
-      y_r_ -= *v_;
+      y_r.Add(-1.0, *v_);
       Op_Imag_->Mult(x_r, *v_);
-      y_i_ += *v_;
+      y_i.Add(1.0, *v_);
    }
 
    if (convention_ == BLOCK_SYMMETRIC)
    {
-      y_i_ *= -1.0;
+      y_i *= -1.0;
    }
 }
 
 void ComplexOperator::MultTranspose(const Vector &x, Vector &y) const
 {
-   double * x_data = x.GetData();
-   y_r_.SetData(x_data);
-   y_i_.SetData(&x_data[height / 2]);
+   x.Read();
+   y.UseDevice(true); y = 0.0;
 
-   x_r_.SetData(&y[0]);
-   x_i_.SetData(&y[width / 2]);
+   x_r_.MakeRef(const_cast<Vector&>(x), 0, height/2);
+   x_i_.MakeRef(const_cast<Vector&>(x), height/2, height/2);
 
-   this->MultTranspose(y_r_, y_i_, x_r_, x_i_);
+   y_r_.MakeRef(y, 0, width/2);
+   y_i_.MakeRef(y, width/2, width/2);
+
+   this->MultTranspose(x_r_, x_i_, y_r_, y_i_);
+
+   y_r_.SyncAliasMemory(y);
+   y_i_.SyncAliasMemory(y);
+
+   // Destroy alias vectors to prevent dangling aliases when the base vectors
+   // are deleted
+   x_r_.Destroy();
+   x_i_.Destroy();
+   y_r_.Destroy();
+   y_i_.Destroy();
 }
 
 void ComplexOperator::MultTranspose(const Vector &x_r, const Vector &x_i,
@@ -136,13 +164,17 @@ void ComplexOperator::MultTranspose(const Vector &x_r, const Vector &x_i,
       y_r = 0.0;
       y_i = 0.0;
    }
+
    if (Op_Imag_)
    {
-      if (!u_) { u_ = new Vector(Op_Imag_->Width()); }
+      if (!u_) { u_ = new Vector(); }
+      u_->UseDevice(true);
+      u_->SetSize(Op_Imag_->Width());
+
       Op_Imag_->MultTranspose(x_i, *u_);
-      y_r_.Add(convention_ == BLOCK_SYMMETRIC ? -1.0 : 1.0, *u_);
+      y_r.Add(convention_ == BLOCK_SYMMETRIC ? -1.0 : 1.0, *u_);
       Op_Imag_->MultTranspose(x_r, *u_);
-      y_i_ -= *u_;
+      y_i.Add(-1.0, *u_);
    }
 }
 

linalg/hypre.cpp

@@ -1048,36 +1048,6 @@ HYPRE_Int HypreParMatrix::MultTranspose(HypreParVector & x, HypreParVector & y,
    return hypre_ParCSRMatrixMatvecT(a, A, x, b, y);
 }
 
-void HypreParMatrix::AbsMult(double a, const Vector &x,
-                             double b, Vector &y) const
-{
-   MFEM_ASSERT(x.Size() == Width(), "invalid x.Size() = " << x.Size()
-               << ", expected size = " << Width());
-   MFEM_ASSERT(y.Size() == Height(), "invalid y.Size() = " << y.Size()
-               << ", expected size = " << Height());
-
-   auto x_data = x.HostRead();
-   auto y_data = (b == 0.0) ? y.HostWrite() : y.HostReadWrite();
-
-   internal::hypre_ParCSRMatrixAbsMatvec(A, a, const_cast<double*>(x_data),
-                                         b, y_data);
-}
-
-void HypreParMatrix::AbsMultTranspose(double a, const Vector &x,
-                                      double b, Vector &y) const
-{
-   MFEM_ASSERT(x.Size() == Height(), "invalid x.Size() = " << x.Size()
-               << ", expected size = " << Height());
-   MFEM_ASSERT(y.Size() == Width(), "invalid y.Size() = " << y.Size()
-               << ", expected size = " << Width());
-
-   auto x_data = x.HostRead();
-   auto y_data = (b == 0.0) ? y.HostWrite() : y.HostReadWrite();
-
-   internal::hypre_ParCSRMatrixAbsMatvecT(A, a, const_cast<double*>(x_data),
-                                          b, y_data);
-}
-
 HypreParMatrix* HypreParMatrix::LeftDiagMult(const SparseMatrix &D,
                                              HYPRE_Int* row_starts) const
 {

linalg/hypre.hpp

@@ -446,12 +446,6 @@ public:
    virtual void MultTranspose(const Vector &x, Vector &y) const
    { MultTranspose(1.0, x, 0.0, y); }
 
-   /// Computes y = a * |A| * x + b * y, using entry-wise absolute values of matrix A
-   void AbsMult(double a, const Vector &x, double b, Vector &y) const;
-
-   /// Computes y = a * |At| * x + b * y, using entry-wise absolute values of the transpose of matrix A
-   void AbsMultTranspose(double a, const Vector &x, double b, Vector &y) const;
-
    /** The "Boolean" analog of y = alpha * A * x + beta * y, where elements in
        the sparsity pattern of the matrix are treated as "true". */
    void BooleanMult(int alpha, const int *x, int beta, int *y)

linalg/hypre_parcsr.cpp

@@ -16,7 +16,6 @@
 
 #include "hypre_parcsr.hpp"
 #include <limits>
-#include <cmath>
 
 namespace mfem
 {
@@ -978,196 +977,6 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
    }
 }
 
-/* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */
-void hypre_CSRMatrixAbsMatvec(hypre_CSRMatrix *A,
-                              HYPRE_Real alpha,
-                              HYPRE_Real *x,
-                              HYPRE_Real beta,
-                              HYPRE_Real *y)
-{
-   HYPRE_Real       *A_data   = hypre_CSRMatrixData(A);
-   HYPRE_Int        *A_i      = hypre_CSRMatrixI(A);
-   HYPRE_Int        *A_j      = hypre_CSRMatrixJ(A);
-   HYPRE_Int         num_rows = hypre_CSRMatrixNumRows(A);
-
-   HYPRE_Int        *A_rownnz = hypre_CSRMatrixRownnz(A);
-   HYPRE_Int         num_rownnz = hypre_CSRMatrixNumRownnz(A);
-
-   HYPRE_Real       *x_data = x;
-   HYPRE_Real       *y_data = y;
-
-   HYPRE_Real        temp, tempx;
-
-   HYPRE_Int         i, jj;
-
-   HYPRE_Int         m;
-
-   HYPRE_Real        xpar=0.7;
-
-   /*-----------------------------------------------------------------------
-    * Do (alpha == 0.0) computation - RDF: USE MACHINE EPS
-    *-----------------------------------------------------------------------*/
-
-   if (alpha == 0.0)
-   {
-      for (i = 0; i < num_rows; i++)
-      {
-         y_data[i] *= beta;
-      }
-      return;
-   }
-
-   /*-----------------------------------------------------------------------
-    * y = (beta/alpha)*y
-    *-----------------------------------------------------------------------*/
-
-   temp = beta / alpha;
-
-   if (temp != 1.0)
-   {
-      if (temp == 0.0)
-      {
-         for (i = 0; i < num_rows; i++)
-         {
-            y_data[i] = 0.0;
-         }
-      }
-      else
-      {
-         for (i = 0; i < num_rows; i++)
-         {
-            y_data[i] *= temp;
-         }
-      }
-   }
-
-   /*-----------------------------------------------------------------
-    * y += abs(A)*x
-    *-----------------------------------------------------------------*/
-
-   /* use rownnz pointer to do the abs(A)*x multiplication
-      when num_rownnz is smaller than num_rows */
-
-   if (num_rownnz < xpar*(num_rows))
-   {
-      for (i = 0; i < num_rownnz; i++)
-      {
-         m = A_rownnz[i];
-
-         tempx = 0;
-         for (jj = A_i[m]; jj < A_i[m+1]; jj++)
-         {
-            tempx += std::abs(A_data[jj])*x_data[A_j[jj]];
-         }
-         y_data[m] += tempx;
-      }
-   }
-   else
-   {
-      for (i = 0; i < num_rows; i++)
-      {
-         tempx = 0;
-         for (jj = A_i[i]; jj < A_i[i+1]; jj++)
-         {
-            tempx += std::abs(A_data[jj])*x_data[A_j[jj]];
-         }
-         y_data[i] += tempx;
-      }
-   }
-
-   /*-----------------------------------------------------------------
-    * y = alpha*y
-    *-----------------------------------------------------------------*/
-
-   if (alpha != 1.0)
-   {
-      for (i = 0; i < num_rows; i++)
-      {
-         y_data[i] *= alpha;
-      }
-   }
-
-}
-
-/* Based on hypre_CSRMatrixMatvecT in hypre's csr_matvec.c */
-void hypre_CSRMatrixAbsMatvecT(hypre_CSRMatrix *A,
-                               HYPRE_Real alpha,
-                               HYPRE_Real *x,
-                               HYPRE_Real beta,
-                               HYPRE_Real *y)
-{
-   HYPRE_Real       *A_data   = hypre_CSRMatrixData(A);
-   HYPRE_Int        *A_i      = hypre_CSRMatrixI(A);
-   HYPRE_Int        *A_j      = hypre_CSRMatrixJ(A);
-   HYPRE_Int         num_rows = hypre_CSRMatrixNumRows(A);
-   HYPRE_Int         num_cols = hypre_CSRMatrixNumCols(A);
-
-   HYPRE_Real       *x_data = x;
-   HYPRE_Real       *y_data = y;
-
-   HYPRE_Int         i, j, jj;
-
-   HYPRE_Real        temp;
-
-   if (alpha == 0.0)
-   {
-      for (i = 0; i < num_cols; i++)
-      {
-         y_data[i] *= beta;
-      }
-      return;
-   }
-
-   /*-----------------------------------------------------------------------
-    * y = (beta/alpha)*y
-    *-----------------------------------------------------------------------*/
-
-   temp = beta / alpha;
-
-   if (temp != 1.0)
-   {
-      if (temp == 0.0)
-      {
-         for (i = 0; i < num_cols; i++)
-         {
-            y_data[i] = 0.0;
-         }
-      }
-      else
-      {
-         for (i = 0; i < num_cols; i++)
-         {
-            y_data[i] *= temp;
-         }
-      }
-   }
-
-   /*-----------------------------------------------------------------
-    * y += abs(A)^T*x
-    *-----------------------------------------------------------------*/
-
-   for (i = 0; i < num_rows; i++)
-   {
-      for (jj = A_i[i]; jj < A_i[i+1]; jj++)
-      {
-         j = A_j[jj];
-         y_data[j] += std::abs(A_data[jj]) * x_data[i];
-      }
-   }
-
-   /*-----------------------------------------------------------------
-    * y = alpha*y
-    *-----------------------------------------------------------------*/
-
-   if (alpha != 1.0)
-   {
-      for (i = 0; i < num_cols; i++)
-      {
-         y_data[i] *= alpha;
-      }
-   }
-}
-
 /* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */
 void hypre_CSRMatrixBooleanMatvec(hypre_CSRMatrix *A,
                                   HYPRE_Bool alpha,
@@ -1427,143 +1236,6 @@ hypre_ParCSRCommHandleCreate_bool(HYPRE_Int            job,
    return comm_handle;
 }
 
-/* Based on hypre_ParCSRMatrixMatvec in par_csr_matvec.c */
-void hypre_ParCSRMatrixAbsMatvec(hypre_ParCSRMatrix *A,
-                                 HYPRE_Real alpha,
-                                 HYPRE_Real *x,
-                                 HYPRE_Real beta,
-                                 HYPRE_Real *y)
-{
-   hypre_ParCSRCommHandle *comm_handle;
-   hypre_ParCSRCommPkg *comm_pkg = hypre_ParCSRMatrixCommPkg(A);
-   hypre_CSRMatrix   *diag   = hypre_ParCSRMatrixDiag(A);
-   hypre_CSRMatrix   *offd   = hypre_ParCSRMatrixOffd(A);
-
-   HYPRE_Int          num_cols_offd = hypre_CSRMatrixNumCols(offd);
-   HYPRE_Int          num_sends, i, j, index;
-
-   HYPRE_Real        *x_tmp, *x_buf;
-
-   x_tmp = mfem_hypre_CTAlloc(HYPRE_Real, num_cols_offd);
-
-   /*---------------------------------------------------------------------
-    * If there exists no CommPkg for A, a CommPkg is generated using
-    * equally load balanced partitionings
-    *--------------------------------------------------------------------*/
-   if (!comm_pkg)
-   {
-      hypre_MatvecCommPkgCreate(A);
-      comm_pkg = hypre_ParCSRMatrixCommPkg(A);
-   }
-
-   num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-   x_buf = mfem_hypre_CTAlloc(
-              HYPRE_Real, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
-
-   index = 0;
-   for (i = 0; i < num_sends; i++)
-   {
-      j = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
-      for ( ; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
-      {
-         x_buf[index++] = x[hypre_ParCSRCommPkgSendMapElmt(comm_pkg, j)];
-      }
-   }
-
-   comm_handle = hypre_ParCSRCommHandleCreate(1, comm_pkg, x_buf, x_tmp);
-
-   hypre_CSRMatrixAbsMatvec(diag, alpha, x, beta, y);
-
-   hypre_ParCSRCommHandleDestroy(comm_handle);
-
-   if (num_cols_offd)
-   {
-      hypre_CSRMatrixAbsMatvec(offd, alpha, x_tmp, 1.0, y);
-   }
-
-   mfem_hypre_TFree(x_buf);
-   mfem_hypre_TFree(x_tmp);
-}
-
-/* Based on hypre_ParCSRMatrixMatvecT in par_csr_matvec.c */
-void hypre_ParCSRMatrixAbsMatvecT(hypre_ParCSRMatrix *A,
-                                  HYPRE_Real alpha,
-                                  HYPRE_Real *x,
-                                  HYPRE_Real beta,
-                                  HYPRE_Real *y)
-{
-   hypre_ParCSRCommHandle *comm_handle;
-   hypre_ParCSRCommPkg    *comm_pkg = hypre_ParCSRMatrixCommPkg(A);
-   hypre_CSRMatrix        *diag = hypre_ParCSRMatrixDiag(A);
-   hypre_CSRMatrix        *offd = hypre_ParCSRMatrixOffd(A);
-   HYPRE_Real             *y_tmp;
-   HYPRE_Real             *y_buf;
-
-   HYPRE_Int               num_cols_offd = hypre_CSRMatrixNumCols(offd);
-
-   HYPRE_Int               i, j, jj, end, num_sends;
-
-   y_tmp = mfem_hypre_TAlloc(HYPRE_Real, num_cols_offd);
-
-   /*---------------------------------------------------------------------
-    * If there exists no CommPkg for A, a CommPkg is generated using
-    * equally load balanced partitionings
-    *--------------------------------------------------------------------*/
-   if (!comm_pkg)
-   {
-      hypre_MatvecCommPkgCreate(A);
-      comm_pkg = hypre_ParCSRMatrixCommPkg(A);
-   }
-
-   num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
-   y_buf = mfem_hypre_CTAlloc(
-              HYPRE_Real, hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends));
-
-   if (num_cols_offd)
-   {
-#if MFEM_HYPRE_VERSION >= 21100
-      if (A->offdT)
-      {
-         // offdT is optional. Used only if it's present.
-         hypre_CSRMatrixAbsMatvec(A->offdT, alpha, x, 0., y_tmp);
-      }
-      else
-#endif
-      {
-         hypre_CSRMatrixAbsMatvecT(offd, alpha, x, 0., y_tmp);
-      }
-   }
-
-   comm_handle = hypre_ParCSRCommHandleCreate(2, comm_pkg, y_tmp, y_buf);
-
-#if MFEM_HYPRE_VERSION >= 21100
-   if (A->diagT)
-   {
-      // diagT is optional. Used only if it's present.
-      hypre_CSRMatrixAbsMatvec(A->diagT, alpha, x, beta, y);
-   }
-   else
-#endif
-   {
-      hypre_CSRMatrixAbsMatvecT(diag, alpha, x, beta, y);
-   }
-
-   hypre_ParCSRCommHandleDestroy(comm_handle);
-
-   for (i = 0; i < num_sends; i++)
-   {
-      end = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1);
-      for (j = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i); j < end; j++)
-      {
-         jj = hypre_ParCSRCommPkgSendMapElmt(comm_pkg, j);
-         y[jj] += y_buf[j];
-      }
-   }
-
-   mfem_hypre_TFree(y_buf);
-   mfem_hypre_TFree(y_tmp);
-}
-
 /* Based on hypre_ParCSRMatrixMatvec in par_csr_matvec.c */
 void hypre_ParCSRMatrixBooleanMatvec(hypre_ParCSRMatrix *A,
                                      HYPRE_Bool alpha,

linalg/hypre_parcsr.hpp

@@ -118,34 +118,6 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
 typedef int HYPRE_Bool;
 #define HYPRE_MPI_BOOL MPI_INT
 
-/// Computes y = alpha * |A| * x + beta * y, using entry-wise absolute values of matrix A
-void hypre_CSRMatrixAbsMatvec(hypre_CSRMatrix *A,
-                              HYPRE_Real alpha,
-                              HYPRE_Real *x,
-                              HYPRE_Real beta,
-                              HYPRE_Real *y);
-
-/// Computes y = alpha * |At| * x + beta * y, using entry-wise absolute values of the transpose of matrix A
-void hypre_CSRMatrixAbsMatvecT(hypre_CSRMatrix *A,
-                               HYPRE_Real alpha,
-                               HYPRE_Real *x,
-                               HYPRE_Real beta,
-                               HYPRE_Real *y);
-
-/// Computes y = alpha * |A| * x + beta * y, using entry-wise absolute values of matrix A
-void hypre_ParCSRMatrixAbsMatvec(hypre_ParCSRMatrix *A,
-                                 HYPRE_Real alpha,
-                                 HYPRE_Real *x,
-                                 HYPRE_Real beta,
-                                 HYPRE_Real *y);
-
-/// Computes y = alpha * |At| * x + beta * y, using entry-wise absolute values of the transpose of matrix A
-void hypre_ParCSRMatrixAbsMatvecT(hypre_ParCSRMatrix *A,
-                                  HYPRE_Real alpha,
-                                  HYPRE_Real *x,
-                                  HYPRE_Real beta,
-                                  HYPRE_Real *y);
-
 /** The "Boolean" analog of y = alpha * A * x + beta * y, where elements in the
     sparsity pattern of the CSR matrix A are treated as "true". */
 void hypre_CSRMatrixBooleanMatvec(hypre_CSRMatrix *A,

linalg/solvers.cpp

@@ -134,7 +134,7 @@ OperatorJacobiSmoother::OperatorJacobiSmoother(const BilinearForm &a,
 
 OperatorJacobiSmoother::OperatorJacobiSmoother(const Vector &d,
                                                const Array<int> &ess_tdofs,
-                                               const double dmpng)
+                                               const double dmpng, const bool inverse)
    :
    Solver(d.Size()),
    N(d.Size()),
@@ -143,16 +143,30 @@ OperatorJacobiSmoother::OperatorJacobiSmoother(const Vector &d,
    ess_tdof_list(ess_tdofs),
    residual(N)
 {
-   Setup(d);
+   Setup(d, inverse);
 }
 
-void OperatorJacobiSmoother::Setup(const Vector &diag)
+void OperatorJacobiSmoother::Setup(const Vector &diag, const bool inverse)
 {
    residual.UseDevice(true);
    const double delta = damping;
    auto D = diag.Read();
    auto DI = dinv.Write();
-   MFEM_FORALL(i, N, DI[i] = delta / D[i]; );
+   if (inverse)
+   {
+      if (delta > 0.0)
+      {
+         MFEM_FORALL(i, N, DI[i] = delta * D[i]; );
+      }
+      else
+      {
+         MFEM_FORALL(i, N, DI[i] = D[i]; );
+      }
+   }
+   else
+   {
+      MFEM_FORALL(i, N, DI[i] = delta / D[i]; );
+   }
    auto I = ess_tdof_list.Read();
    MFEM_FORALL(i, ess_tdof_list.Size(), DI[I[i]] = delta; );
 }
@@ -2284,7 +2298,7 @@ void MinimumDiscardedFillOrdering(SparseMatrix &C, Array<int> &p)
       {
          int i = J[ii];
          // Find value of (i,k)
-         double C_ik = 0.0;
+         double C_ik;
          for (int kk=I[i]; kk<I[i+1]; ++kk)
          {
             if (J[kk] == k)
@@ -2334,7 +2348,7 @@ void MinimumDiscardedFillOrdering(SparseMatrix &C, Array<int> &p)
             int i = J[ii2];
             if (w_heap.picked(i)) { continue; }
             // Find value of (i,k)
-            double C_ik = 0.0;
+            double C_ik;
             for (int kk2=I[i]; kk2<I[i+1]; ++kk2)
             {
                if (J[kk2] == k)
@@ -2665,42 +2679,6 @@ void BlockILU::Mult(const Vector &b, Vector &x) const
    }
 }
 
-
-void ResidualBCMonitor::MonitorResidual(
-   int it, double norm, const Vector &r, bool final)
-{
-   if (!ess_dofs_list) { return; }
-
-   double bc_norm_squared = 0.0;
-   r.HostRead();
-   ess_dofs_list->HostRead();
-   for (int i = 0; i < ess_dofs_list->Size(); i++)
-   {
-      const double r_entry = r((*ess_dofs_list)[i]);
-      bc_norm_squared += r_entry*r_entry;
-   }
-   bool print = true;
-#ifdef MFEM_USE_MPI
-   MPI_Comm comm = iter_solver->GetComm();
-   if (comm != MPI_COMM_NULL)
-   {
-      double glob_bc_norm_squared = 0.0;
-      MPI_Reduce(&bc_norm_squared, &glob_bc_norm_squared, 1, MPI_DOUBLE,
-                 MPI_SUM, 0, comm);
-      bc_norm_squared = glob_bc_norm_squared;
-      int rank;
-      MPI_Comm_rank(comm, &rank);
-      print = (rank == 0);
-   }
-#endif
-   if ((it == 0 || final || bc_norm_squared > 0.0) && print)
-   {
-      mfem::out << "      ResidualBCMonitor : b.c. residual norm = "
-                << sqrt(bc_norm_squared) << endl;
-   }
-}
-
-
 #ifdef MFEM_USE_SUITESPARSE
 
 void UMFPackSolver::Init()
@@ -2972,36 +2950,4 @@ KLUSolver::~KLUSolver()
 
 #endif // MFEM_USE_SUITESPARSE
 
-IncompleteCholesky::IncompleteCholesky(SparseMatrix &A_) : A(&A_)
-{
-#ifdef MFEM_USE_CUDA
-   A->IncompleteCholeskySetup();
-#endif
-}
-
-void IncompleteCholesky::Mult(const Vector &b, Vector &x) const
-{
-#ifdef MFEM_USE_CUDA
-   A->IncompleteCholeskyMult(b, x);
-#else
-   x = b;
-#endif
-}
-
-ILUcusparse::ILUcusparse(SparseMatrix &A_) : A(&A_)
-{
-#ifdef MFEM_USE_CUDA
-   A->ILUSetup();
-#endif
-}
-
-void ILUcusparse::Mult(const Vector &b, Vector &x) const
-{
-#ifdef MFEM_USE_CUDA
-   A->ILUMult(b, x);
-#else
-   x = b;
-#endif
-}
-
 }

linalg/solvers.hpp

@@ -33,12 +33,8 @@ class BilinearForm;
 /// Abstract base class for an iterative solver monitor
 class IterativeSolverMonitor
 {
-protected:
-   /// The last IterativeSolver to which this monitor was attached.
-   const class IterativeSolver *iter_solver;
-
 public:
-   IterativeSolverMonitor() : iter_solver(nullptr) {}
+   IterativeSolverMonitor() {}
 
    virtual ~IterativeSolverMonitor() {}
 
@@ -53,11 +49,6 @@ public:
                                 bool final)
    {
    }
-
-   /** @brief This method is invoked by ItertiveSolver::SetMonitor, informing
-       the monitor which IterativeSolver is using it. */
-   void SetIterativeSolver(const IterativeSolver &solver)
-   { iter_solver = &solver; }
 };
 
 /// Abstract base class for iterative solver
@@ -109,15 +100,7 @@ public:
    virtual void SetOperator(const Operator &op);
 
    /// Set the iterative solver monitor
-   void SetMonitor(IterativeSolverMonitor &m)
-   { monitor = &m; m.SetIterativeSolver(*this); }
-
-#ifdef MFEM_USE_MPI
-   /** @brief Return the associated MPI communicator, or MPI_COMM_NULL if no
-       communicator is set. */
-   MPI_Comm GetComm() const
-   { return dot_prod_type == 0 ? MPI_COMM_NULL : comm; }
-#endif
+   void SetMonitor(IterativeSolverMonitor &m) { monitor = &m; }
 };
 
 
@@ -142,13 +125,14 @@ public:
        the matrix-free setting. */
    OperatorJacobiSmoother(const Vector &d,
                           const Array<int> &ess_tdof_list,
-                          const double damping=1.0);
+                          const double damping=1.0, 
+                          const bool inverse=false);
    ~OperatorJacobiSmoother() {}
 
    void Mult(const Vector &x, Vector &y) const;
    void MultTranspose(const Vector &x, Vector &y) const { Mult(x, y); }
    void SetOperator(const Operator &op) { oper = &op; }
-   void Setup(const Vector &diag);
+   void Setup(const Vector &diag, const bool inverse=false);
 
 private:
    const int N;
@@ -706,25 +690,6 @@ private:
    mutable Array<int> ipiv;
 };
 
-
-/// Monitor that checks whether the residual is zero at a given set of dofs.
-/** This monitor is useful for checking if the initial guess, rhs, operator, and
-    preconditioner are properly setup for solving in the subspace with imposed
-    essential boundary conditions. */
-class ResidualBCMonitor : public IterativeSolverMonitor
-{
-protected:
-   const Array<int> *ess_dofs_list; ///< Not owned
-
-public:
-   ResidualBCMonitor(const Array<int> &ess_dofs_list_)
-      : ess_dofs_list(&ess_dofs_list_) { }
-
-   void MonitorResidual(int it, double norm, const Vector &r,
-                        bool final) override;
-};
-
-
 #ifdef MFEM_USE_SUITESPARSE
 
 /// Direct sparse solver using UMFPACK
@@ -799,28 +764,6 @@ public:
 
 #endif // MFEM_USE_SUITESPARSE
 
-class IncompleteCholesky : public IterativeSolver
-{
-private:
-   SparseMatrix *A;
-
-public:
-   IncompleteCholesky(SparseMatrix &A_);
-
-   virtual void Mult(const Vector &b, Vector &x) const;
-};
-
-class ILUcusparse : public IterativeSolver
-{
-private:
-   SparseMatrix *A;
-
-public:
-   ILUcusparse(SparseMatrix &A_);
-
-   virtual void Mult(const Vector &b, Vector &x) const;
-};
-
 }
 
 #endif // MFEM_SOLVERS

linalg/sparsemat.cpp

@@ -28,25 +28,6 @@ namespace mfem
 
 using namespace std;
 
-#ifdef MFEM_USE_CUDA
-int SparseMatrix::SparseMatrixCount = 0;
-cusparseHandle_t SparseMatrix::handle;
-size_t SparseMatrix::bufferSize = 0;
-void * SparseMatrix::dBuffer = nullptr;
-#endif
-
-void SparseMatrix::InitCuSparse()
-{
-   // Initialize cuSPARSE library
-#ifdef MFEM_USE_CUDA
-   SparseMatrixCount++;
-   if (SparseMatrixCount == 1 && Device::Allows(Backend::CUDA_MASK))
-   {
-      cusparseCreate(&handle);
-   }
-#endif
-}
-
 SparseMatrix::SparseMatrix(int nrows, int ncols)
    : AbstractSparseMatrix(nrows, (ncols >= 0) ? ncols : nrows),
      Rows(new RowNode *[nrows]),
@@ -69,8 +50,6 @@ SparseMatrix::SparseMatrix(int nrows, int ncols)
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = new RowNodeAlloc;
 #endif
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
@@ -88,8 +67,6 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n)
 #ifdef MFEM_USE_MEMALLOC
    NodesMem = NULL;
 #endif
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
@@ -121,8 +98,6 @@ SparseMatrix::SparseMatrix(int *i, int *j, double *data, int m, int n,
          A[i] = 0.0;
       }
    }
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
@@ -144,8 +119,6 @@ SparseMatrix::SparseMatrix(int nrows, int ncols, int rowsize)
    {
       I[i] = i * rowsize;
    }
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
@@ -211,8 +184,6 @@ SparseMatrix::SparseMatrix(const SparseMatrix &mat, bool copy_graph)
    ColPtrNode = NULL;
    At = NULL;
    isSorted = mat.isSorted;
-
-   InitCuSparse();
 }
 
 SparseMatrix::SparseMatrix(const Vector &v)
@@ -240,8 +211,6 @@ SparseMatrix::SparseMatrix(const Vector &v)
       J[r] = r;
       A[r] = v[r];
    }
-
-   InitCuSparse();
 }
 
 SparseMatrix& SparseMatrix::operator=(const SparseMatrix &rhs)
@@ -281,16 +250,6 @@ void SparseMatrix::SetEmpty()
    NodesMem = NULL;
 #endif
    isSorted = false;
-
-#ifdef MFEM_USE_CUDA
-   if (initBuffers)
-   {
-      cusparseDestroySpMat(matA_descr);
-      cusparseDestroyDnVec(vecX_descr);
-      cusparseDestroyDnVec(vecY_descr);
-      initBuffers = false;
-   }
-#endif
 }
 
 int SparseMatrix::RowSize(const int i) const
@@ -610,7 +569,7 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
       const double *xp = x.HostRead();
       double *yp = y.HostReadWrite();
 
-      // The matrix is not finalized, but multiplication is still possible
+      //  The matrix is not finalized, but multiplication is still possible
       for (int i = 0; i < height; i++)
       {
          RowNode *row = Rows[i];
@@ -633,72 +592,16 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
    auto d_A = Read(A, nnz);
    auto d_x = x.Read();
    auto d_y = y.ReadWrite();
-
-   // Skip if matrix has no non-zeros
-   if (nnz == 0) {return;}
-   if (Device::Allows(Backend::CUDA_MASK) && useCuSparse)
+   MFEM_FORALL(i, height,
    {
-#ifdef MFEM_USE_CUDA
-      const double alpha = a;
-      const double beta  = 1.0;
-
-      // Setup descriptors
-      if (!initBuffers)
+      double d = 0.0;
+      const int end = d_I[i+1];
+      for (int j = d_I[i]; j < end; j++)
       {
-         // Setup matrix descriptor
-         cusparseCreateCsr(&matA_descr,Height(), Width(), J.Capacity(),
-                           const_cast<int *>(d_I),
-                           const_cast<int *>(d_J), const_cast<double *>(d_A), CUSPARSE_INDEX_32I,
-                           CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F);
-
-         // Create handles for input/output vectors
-         cusparseCreateDnVec(&vecX_descr, x.Size(), const_cast<double *>(d_x),
-                             CUDA_R_64F);
-         cusparseCreateDnVec(&vecY_descr, y.Size(), d_y, CUDA_R_64F);
-
-         initBuffers = true;
+         d += d_A[j] * d_x[d_J[j]];
       }
-
-      // Allocate kernel space. Buffer is shared between different sparsemats
-      size_t newBufferSize = 0;
-      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
-                              matA_descr,
-                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
-                              CUSPARSE_CSRMV_ALG1, &newBufferSize);
-
-      // Check if we need to resize
-      if (newBufferSize > bufferSize)
-      {
-         bufferSize = newBufferSize;
-         if (dBuffer != NULL) { CuMemFree(dBuffer); }
-         CuMemAlloc(&dBuffer, bufferSize);
-      }
-
-      // Update input/output vectors
-      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
-      cusparseDnVecSetValues(vecY_descr, d_y);
-
-      // Y = alpha A * X + beta * Y
-      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
-                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, CUSPARSE_CSRMV_ALG1, dBuffer);
-#endif
-   }
-   else
-   {
-      // Native version
-      MFEM_FORALL(i, height,
-      {
-         double d = 0.0;
-         const int end = d_I[i+1];
-         for (int j = d_I[i]; j < end; j++)
-         {
-            d += d_A[j] * d_x[d_J[j]];
-         }
-         d_y[i] += a * d;
-      });
-
-   }
-
+      d_y[i] += a * d;
+   });
 #else
    const double *Ap = A, *xp = x.GetData();
    double *yp = y.GetData();
@@ -881,101 +784,6 @@ void SparseMatrix::BooleanMultTranspose(const Array<int> &x,
    }
 }
 
-void SparseMatrix::AbsMult(const Vector &x, Vector &y) const
-{
-   MFEM_ASSERT(width == x.Size(), "Input vector size (" << x.Size()
-               << ") must match matrix width (" << width << ")");
-   MFEM_ASSERT(height == y.Size(), "Output vector size (" << y.Size()
-               << ") must match matrix height (" << height << ")");
-
-   if (Finalized()) { y.UseDevice(true); }
-   y = 0.0;
-
-   if (!Finalized())
-   {
-      const double *xp = x.HostRead();
-      double *yp = y.HostReadWrite();
-
-      //  The matrix is not finalized, but multiplication is still possible
-      for (int i = 0; i < height; i++)
-      {
-         RowNode *row = Rows[i];
-         double b = 0.0;
-         for ( ; row != NULL; row = row->Prev)
-         {
-            b += std::abs(row->Value) * xp[row->Column];
-         }
-         *yp += b;
-         yp++;
-      }
-      return;
-   }
-
-   const int height = this->height;
-   const int nnz = J.Capacity();
-   auto d_I = Read(I, height+1);
-   auto d_J = Read(J, nnz);
-   auto d_A = Read(A, nnz);
-   auto d_x = x.Read();
-   auto d_y = y.ReadWrite();
-   MFEM_FORALL(i, height,
-   {
-      double d = 0.0;
-      const int end = d_I[i+1];
-      for (int j = d_I[i]; j < end; j++)
-      {
-         d += std::abs(d_A[j]) * d_x[d_J[j]];
-      }
-      d_y[i] += d;
-   });
-}
-
-void SparseMatrix::AbsMultTranspose(const Vector &x, Vector &y) const
-{
-   MFEM_ASSERT(height == x.Size(), "Input vector size (" << x.Size()
-               << ") must match matrix height (" << height << ")");
-   MFEM_ASSERT(width == y.Size(), "Output vector size (" << y.Size()
-               << ") must match matrix width (" << width << ")");
-
-   y = 0.0;
-
-   if (!Finalized())
-   {
-      double *yp = y.GetData();
-      // The matrix is not finalized, but multiplication is still possible
-      for (int i = 0; i < height; i++)
-      {
-         RowNode *row = Rows[i];
-         double b = x(i);
-         for ( ; row != NULL; row = row->Prev)
-         {
-            yp[row->Column] += fabs(row->Value) * b;
-         }
-      }
-      return;
-   }
-
-   if (At)
-   {
-      At->AbsMult(x, y);
-   }
-   else
-   {
-      MFEM_VERIFY(Device::IsDisabled(), "transpose action on device is not "
-                  "enabled; see BuildTranspose() for details.");
-      for (int i = 0; i < height; i++)
-      {
-         const double xi = x[i];
-         const int end = I[i+1];
-         for (int j = I[i]; j < end; j++)
-         {
-            const int Jj = J[j];
-            y[Jj] += std::abs(A[j]) * xi;
-         }
-      }
-   }
-}
-
 double SparseMatrix::InnerProduct(const Vector &x, const Vector &y) const
 {
    MFEM_ASSERT(x.Size() == Width(), "x.Size() = " << x.Size()
@@ -3154,16 +2962,6 @@ void SparseMatrix::Destroy()
    delete NodesMem;
 #endif
    delete At;
-
-#ifdef MFEM_USE_CUDA
-   if (initBuffers)
-   {
-      cusparseDestroySpMat(matA_descr);
-      cusparseDestroyDnVec(vecX_descr);
-      cusparseDestroyDnVec(vecY_descr);
-      initBuffers = false;
-   }
-#endif
 }
 
 int SparseMatrix::ActualWidth() const
@@ -3899,323 +3697,4 @@ void SparseMatrix::Swap(SparseMatrix &other)
    mfem::Swap(isSorted, other.isSorted);
 }
 
-#ifdef MFEM_USE_CUDA
-void SparseMatrix::IncompleteCholeskyMult(const Vector &x, Vector &y) const
-{
-   if (!(Device::Allows(Backend::CUDA_MASK) && useCuSparse))
-   {
-      y = x;
-      return;
-   }
-
-   MFEM_VERIFY(initCholesky, "Setup not done");
-
-   const double alpha = 1.0;
-
-   auto d_x = x.Read();
-   auto d_y = y.ReadWrite();
-   auto d_z = vecZ.ReadWrite();
-
-   const int height = this->height;
-   const int nnz = J.Capacity();
-   int64_t m = height;
-
-   auto d_csrRowPtr = Read(I, height+1);
-   auto d_csrColInd = Read(J, nnz);
-   auto d_csrVal = Read(A, nnz);
-
-   cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
-   cusparseDnVecSetValues(vecY_descr, d_y);
-   cusparseDnVecSetValues(vecZ_descr, d_z);
-
-   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
-   const cusparseOperation_t trans_Lt = CUSPARSE_OPERATION_TRANSPOSE;
-
-   const cusparseSolvePolicy_t policy_L  = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
-   const cusparseSolvePolicy_t policy_Lt = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
-
-   // Solve L*z = x
-   cusparseDcsrsv2_solve(handle, trans_L, m, nnz, &alpha, descr_L,
-                         d_csrVal, d_csrRowPtr, d_csrColInd, info_L,
-                         d_x, d_z, policy_L, pBuffer);
-
-   // Solve L'*y = z
-   cusparseDcsrsv2_solve(handle, trans_Lt, m, nnz, &alpha, descr_L,
-                         d_csrVal, d_csrRowPtr, d_csrColInd, info_Lt,
-                         d_z, d_y, policy_Lt, pBuffer);
-}
-
-void SparseMatrix::IncompleteCholeskySetup()
-{
-   if (!(Device::Allows(Backend::CUDA_MASK) && useCuSparse))
-   {
-      return;
-   }
-
-   MFEM_VERIFY(!initILU && !initCholesky, "");
-
-   const int height = this->height;
-   const int nnz = J.Capacity();
-
-   auto d_csrRowPtr = Read(I, height+1);
-   auto d_csrColInd = Read(J, nnz);
-   auto d_csrVal = Read(A, nnz);
-
-   //MFEM_VERIFY(I[0] == 0, "cusparse thinks this is not zero based");
-
-   csric02Info_t info_M  = 0;
-   int bufferSize_M;
-   int pBufferSize_L;
-   int pBufferSize_Lt;
-   int pBufferSize;
-   int structural_zero;
-   int numerical_zero;
-
-   const cusparseSolvePolicy_t policy_M  = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
-   const cusparseSolvePolicy_t policy_L  = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
-   const cusparseSolvePolicy_t policy_Lt = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
-   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
-   const cusparseOperation_t trans_Lt = CUSPARSE_OPERATION_TRANSPOSE;
-
-   // step 1: create a descriptor which contains
-   // - matrix M is base-0
-   // - matrix L is base-0
-   // - matrix L is lower triangular
-   // - matrix L has non-unit diagonal
-   cusparseCreateMatDescr(&descr_M);
-   cusparseSetMatIndexBase(descr_M, CUSPARSE_INDEX_BASE_ZERO);
-   cusparseSetMatType(descr_M, CUSPARSE_MATRIX_TYPE_GENERAL);
-
-   cusparseCreateMatDescr(&descr_L);
-   cusparseSetMatIndexBase(descr_L, CUSPARSE_INDEX_BASE_ZERO);
-   cusparseSetMatType(descr_L, CUSPARSE_MATRIX_TYPE_GENERAL);
-   cusparseSetMatFillMode(descr_L, CUSPARSE_FILL_MODE_LOWER);
-   cusparseSetMatDiagType(descr_L, CUSPARSE_DIAG_TYPE_NON_UNIT);
-
-   // step 2: create a empty info structure
-   // we need one info for csric02 and two info's for csrsv2
-   cusparseCreateCsric02Info(&info_M);
-   cusparseCreateCsrsv2Info(&info_L);
-   cusparseCreateCsrsv2Info(&info_Lt);
-
-   int64_t m = height;
-
-   // step 3: query how much memory used in csric02 and csrsv2, and allocate the buffer
-   cusparseDcsric02_bufferSize(handle, m, nnz,
-                               descr_M, const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                               const_cast<int *>(d_csrColInd), info_M, &bufferSize_M);
-   cusparseDcsrsv2_bufferSize(handle, trans_L, m, nnz,
-                              descr_L, const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                              const_cast<int *>(d_csrColInd), info_L, &pBufferSize_L);
-   cusparseDcsrsv2_bufferSize(handle, trans_Lt, m, nnz,
-                              descr_L, const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                              const_cast<int *>(d_csrColInd), info_Lt,&pBufferSize_Lt);
-
-   pBufferSize = max(bufferSize_M, max(pBufferSize_L, pBufferSize_Lt));
-
-   // pBuffer returned by cudaMalloc is automatically aligned to 128 bytes.
-   cudaMalloc((void**)&pBuffer, pBufferSize);
-
-   // step 4: perform analysis of incomplete Cholesky on M
-   //         perform analysis of triangular solve on L
-   //         perform analysis of triangular solve on L'
-   // The lower triangular part of M has the same sparsity pattern as L, so
-   // we can do analysis of csric02 and csrsv2 simultaneously.
-
-   cusparseDcsric02_analysis(handle, m, nnz, descr_M,
-                             d_csrVal, d_csrRowPtr, d_csrColInd, info_M,
-                             policy_M, pBuffer);
-   status = cusparseXcsric02_zeroPivot(handle, info_M, &structural_zero);
-   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
-   {
-      printf("A(%d,%d) is missing\n", structural_zero, structural_zero);
-   }
-
-   cusparseDcsrsv2_analysis(handle, trans_L, m, nnz, descr_L,
-                            d_csrVal, d_csrRowPtr, d_csrColInd,
-                            info_L, policy_L, pBuffer);
-
-   cusparseDcsrsv2_analysis(handle, trans_Lt, m, nnz, descr_L,
-                            d_csrVal, d_csrRowPtr, d_csrColInd,
-                            info_Lt, policy_Lt, pBuffer);
-
-   // step 5: M = L * L'
-   cusparseDcsric02(handle, m, nnz, descr_M,
-                    const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                    const_cast<int *>(d_csrColInd), info_M, policy_M, pBuffer);
-   status = cusparseXcsric02_zeroPivot(handle, info_M, &numerical_zero);
-   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
-   {
-      printf("L(%d,%d) is zero\n", numerical_zero, numerical_zero);
-   }
-
-   vecZ.SetSize(height);
-   vecZ = 0.0;
-   auto d_z = vecZ.ReadWrite();
-   cusparseCreateDnVec(&vecZ_descr, vecZ.Size(), d_z, CUDA_R_64F);
-
-   initCholesky = true;
-}
-
-void SparseMatrix::ILUMult(const Vector &x, Vector &y) const
-{
-   MFEM_VERIFY(initILU, "Setup not done");
-
-   const double alpha = 1.0;
-
-   auto d_x = x.Read();
-   auto d_y = y.ReadWrite();
-   auto d_z = vecZ.ReadWrite();
-
-   const int height = this->height;
-   const int nnz = J.Capacity();
-   int64_t m = height;
-
-   auto d_csrRowPtr = Read(I, height+1);
-   auto d_csrColInd = Read(J, nnz);
-   auto d_csrVal = Read(A, nnz);
-
-   cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
-   cusparseDnVecSetValues(vecY_descr, d_y);
-   cusparseDnVecSetValues(vecZ_descr, d_z);
-
-   const cusparseSolvePolicy_t policy_L = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
-   const cusparseSolvePolicy_t policy_U = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
-   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
-   const cusparseOperation_t trans_U  = CUSPARSE_OPERATION_NON_TRANSPOSE;
-
-   // Solve L*z = x
-   cusparseDcsrsv2_solve(handle, trans_L, m, nnz, &alpha, descr_L,
-                         d_csrVal, d_csrRowPtr, d_csrColInd, info_L,
-                         d_x, d_z, policy_L, pBuffer);
-
-   // Solve U*y = z
-   cusparseDcsrsv2_solve(handle, trans_U, m, nnz, &alpha, descr_U,
-                         d_csrVal, d_csrRowPtr, d_csrColInd, info_U,
-                         d_z, d_y, policy_U, pBuffer);
-
-   // TODO: destructor
-}
-
-void SparseMatrix::ILUSetup()
-{
-   if (!(Device::Allows(Backend::CUDA_MASK) && useCuSparse))
-   {
-      return;
-   }
-
-   MFEM_VERIFY(!initILU && !initCholesky, "");
-
-   const int height = this->height;
-   const int nnz = J.Capacity();
-
-   auto d_csrRowPtr = Read(I, height+1);
-   auto d_csrColInd = Read(J, nnz);
-   auto d_csrVal = Read(A, nnz);
-
-   csrilu02Info_t info_M  = 0;
-   int pBufferSize_M;
-   int pBufferSize_L;
-   int pBufferSize_U;
-   int pBufferSize;
-   int structural_zero;
-   int numerical_zero;
-   const cusparseSolvePolicy_t policy_M = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
-   const cusparseSolvePolicy_t policy_L = CUSPARSE_SOLVE_POLICY_NO_LEVEL;
-   const cusparseSolvePolicy_t policy_U = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
-   const cusparseOperation_t trans_L  = CUSPARSE_OPERATION_NON_TRANSPOSE;
-   const cusparseOperation_t trans_U  = CUSPARSE_OPERATION_NON_TRANSPOSE;
-
-   // step 1: create a descriptor which contains
-   // - matrix M is base-0
-   // - matrix L is base-0
-   // - matrix L is lower triangular
-   // - matrix L has unit diagonal
-   // - matrix U is base-0
-   // - matrix U is upper triangular
-   // - matrix U has non-unit diagonal
-   cusparseCreateMatDescr(&descr_M);
-   cusparseSetMatIndexBase(descr_M, CUSPARSE_INDEX_BASE_ZERO);
-   cusparseSetMatType(descr_M, CUSPARSE_MATRIX_TYPE_GENERAL);
-
-   cusparseCreateMatDescr(&descr_L);
-   cusparseSetMatIndexBase(descr_L, CUSPARSE_INDEX_BASE_ZERO);
-   cusparseSetMatType(descr_L, CUSPARSE_MATRIX_TYPE_GENERAL);
-   cusparseSetMatFillMode(descr_L, CUSPARSE_FILL_MODE_LOWER);
-   cusparseSetMatDiagType(descr_L, CUSPARSE_DIAG_TYPE_UNIT);
-
-   cusparseCreateMatDescr(&descr_U);
-   cusparseSetMatIndexBase(descr_U, CUSPARSE_INDEX_BASE_ZERO);
-   cusparseSetMatType(descr_U, CUSPARSE_MATRIX_TYPE_GENERAL);
-   cusparseSetMatFillMode(descr_U, CUSPARSE_FILL_MODE_UPPER);
-   cusparseSetMatDiagType(descr_U, CUSPARSE_DIAG_TYPE_NON_UNIT);
-
-   // step 2: create a empty info structure
-   // we need one info for csrilu02 and two info's for csrsv2
-   cusparseCreateCsrilu02Info(&info_M);
-   cusparseCreateCsrsv2Info(&info_L);
-   cusparseCreateCsrsv2Info(&info_U);
-
-   // step 3: query how much memory used in csrilu02 and csrsv2, and allocate the buffer
-   int64_t m = height;
-
-   cusparseDcsrilu02_bufferSize(handle, m, nnz, descr_M,
-                                const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                                const_cast<int *>(d_csrColInd), info_M, &pBufferSize_M);
-   cusparseDcsrsv2_bufferSize(handle, trans_L, m, nnz, descr_L,
-                              const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                              const_cast<int *>(d_csrColInd), info_L, &pBufferSize_L);
-   cusparseDcsrsv2_bufferSize(handle, trans_U, m, nnz, descr_U,
-                              const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                              const_cast<int *>(d_csrColInd), info_U, &pBufferSize_U);
-
-   pBufferSize = max(pBufferSize_M, max(pBufferSize_L, pBufferSize_U));
-
-   // pBuffer returned by cudaMalloc is automatically aligned to 128 bytes.
-   cudaMalloc((void**)&pBuffer, pBufferSize);
-
-   // step 4: perform analysis of incomplete Cholesky on M
-   //         perform analysis of triangular solve on L
-   //         perform analysis of triangular solve on U
-   // The lower(upper) triangular part of M has the same sparsity pattern as L(U),
-   // we can do analysis of csrilu0 and csrsv2 simultaneously.
-
-   cusparseDcsrilu02_analysis(handle, m, nnz, descr_M,
-                              d_csrVal, d_csrRowPtr, d_csrColInd, info_M,
-                              policy_M, pBuffer);
-   status = cusparseXcsrilu02_zeroPivot(handle, info_M, &structural_zero);
-   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
-   {
-      printf("A(%d,%d) is missing\n", structural_zero, structural_zero);
-   }
-
-   cusparseDcsrsv2_analysis(handle, trans_L, m, nnz, descr_L,
-                            d_csrVal, d_csrRowPtr, d_csrColInd,
-                            info_L, policy_L, pBuffer);
-
-   cusparseDcsrsv2_analysis(handle, trans_U, m, nnz, descr_U,
-                            d_csrVal, d_csrRowPtr, d_csrColInd,
-                            info_U, policy_U, pBuffer);  // bug?
-
-   // step 5: M = L * U
-   cusparseDcsrilu02(handle, m, nnz, descr_M,
-                     const_cast<double *>(d_csrVal), const_cast<int *>(d_csrRowPtr),
-                     const_cast<int *>(d_csrColInd), info_M, policy_M, pBuffer);
-   status = cusparseXcsrilu02_zeroPivot(handle, info_M, &numerical_zero);
-   if (CUSPARSE_STATUS_ZERO_PIVOT == status)
-   {
-      printf("U(%d,%d) is zero\n", numerical_zero, numerical_zero);
-   }
-
-   vecZ.SetSize(height);
-   vecZ = 0.0;
-   auto d_z = vecZ.ReadWrite();
-   cusparseCreateDnVec(&vecZ_descr, vecZ.Size(), d_z, CUDA_R_64F);
-
-   initILU = true;
-
-   // TODO: destructor
-}
-#endif // MFEM_USE_CUDA
-
 }

linalg/sparsemat.hpp

@@ -21,12 +21,6 @@
 #include "../general/globals.hpp"
 #include "densemat.hpp"
 
-#ifdef MFEM_USE_CUDA
-#include <cusparse.h>
-#include <library_types.h>
-#include "../general/cuda.hpp"
-#endif
-
 namespace mfem
 {
 
@@ -86,48 +80,9 @@ protected:
    void Destroy();   // Delete all owned data
    void SetEmpty();  // Init all entries with empty values
 
-   bool useCuSparse{true}; // Use cuSPARSE if available
-
-   // Initialize cuSPARSE
-   void InitCuSparse();
-
-#ifdef MFEM_USE_CUDA
-   cusparseStatus_t status;
-   static cusparseHandle_t handle;
-   cusparseMatDescr_t descr=0;
-   static size_t bufferSize;
-   static void *dBuffer;
-   mutable bool initBuffers{false};
-
-   static int SparseMatrixCount;
-   mutable cusparseSpMatDescr_t matA_descr;
-   mutable cusparseDnVecDescr_t vecX_descr;
-   mutable cusparseDnVecDescr_t vecY_descr;
-   mutable cusparseDnVecDescr_t vecZ_descr;
-   mutable Vector vecZ;
-
-   cusparseMatDescr_t descr_M = 0;
-   cusparseMatDescr_t descr_L = 0;
-   cusparseMatDescr_t descr_U = 0;
-
-   csrsv2Info_t  info_L  = 0;
-   csrsv2Info_t  info_Lt = 0;
-   csrsv2Info_t  info_U  = 0;
-
-   void *pBuffer = 0;
-
-   bool initILU = false;
-   bool initCholesky = false;
-#endif
-
 public:
    /// Create an empty SparseMatrix.
-   SparseMatrix()
-   {
-      SetEmpty();
-
-      InitCuSparse();
-   }
+   SparseMatrix() { SetEmpty(); }
 
    /** @brief Create a sparse matrix with flexible sparsity structure using a
        row-wise linked list (LIL) format. */
@@ -163,8 +118,6 @@ public:
    /// Create a SparseMatrix with diagonal @a v, i.e. A = Diag(v)
    SparseMatrix(const Vector & v);
 
-   // Runtime option to use cuSPARSE. Only valid when using a CUDA backend.
-   void UseCuSparse(bool _useCuSparse = true) { useCuSparse = _useCuSparse;}
 
    /// Assignment operator: deep copy
    SparseMatrix& operator=(const SparseMatrix &rhs);
@@ -355,22 +308,16 @@ public:
 
    /// y = A * x, treating all entries as booleans (zero=false, nonzero=true).
    /** The actual values stored in the data array, #A, are not used - this means
-       that all entries in the sparsity pattern are considered to be true by
+       and that all entries in the sparsity pattern are considered to be true by
        this method. */
    void BooleanMult(const Array<int> &x, Array<int> &y) const;
 
    /// y = At * x, treating all entries as booleans (zero=false, nonzero=true).
    /** The actual values stored in the data array, #A, are not used - this means
-       that all entries in the sparsity pattern are considered to be true by
+       and that all entries in the sparsity pattern are considered to be true by
        this method. */
    void BooleanMultTranspose(const Array<int> &x, Array<int> &y) const;
 
-   /// y = |A| * x, using entry-wise absolute values of matrix A
-   void AbsMult(const Vector &x, Vector &y) const;
-
-   /// y = |At| * x, using entry-wise absolute values of the transpose of matrix A
-   void AbsMultTranspose(const Vector &x, Vector &y) const;
-
    /// Compute y^t A x
    double InnerProduct(const Vector &x, const Vector &y) const;
 
@@ -625,27 +572,8 @@ public:
 
    void Swap(SparseMatrix &other);
 
-#ifdef MFEM_USE_CUDA
-   void IncompleteCholeskySetup();
-   void IncompleteCholeskyMult(const Vector &x, Vector &y) const;
-
-   void ILUSetup();
-   void ILUMult(const Vector &x, Vector &y) const;
-#endif
-
    /// Destroys sparse matrix.
-   virtual ~SparseMatrix()
-   {
-      Destroy();
-#ifdef MFEM_USE_CUDA
-      if (handle && SparseMatrixCount==1 && Device::Allows(Backend::CUDA_MASK))
-      {
-         cusparseDestroy(handle);
-         CuMemFree(dBuffer);
-      }
-      SparseMatrixCount--;
-#endif
-   }
+   virtual ~SparseMatrix() { Destroy(); }
 
    Type GetType() const { return MFEM_SPARSEMAT; }
 };

mesh/CMakeLists.txt

@@ -11,7 +11,6 @@
 
 set(SRCS
   element.cpp
-  gmsh.cpp
   hexahedron.cpp
   mesh.cpp
   mesh_operators.cpp
@@ -30,7 +29,6 @@ set(SRCS
 
 set(HDRS
   element.hpp
-  gmsh.hpp
   hexahedron.hpp
   mesh.hpp
   mesh_headers.hpp

mesh/gmsh.cpp

@@ -1,487 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "gmsh.hpp"
-#include "vtk.hpp"
-
-namespace mfem
-{
-
-int BarycentricToGmshTet(int *b, int ref)
-{
-   int i = b[0];
-   int j = b[1];
-   int k = b[2];
-   int l = b[3];
-   bool ibdr = (i == 0);
-   bool jbdr = (j == 0);
-   bool kbdr = (k == 0);
-   bool lbdr = (l == 0);
-   if (ibdr && jbdr && kbdr)
-   {
-      return 0;
-   }
-   else if (jbdr && kbdr && lbdr)
-   {
-      return 1;
-   }
-   else if (ibdr && kbdr && lbdr)
-   {
-      return 2;
-   }
-   else if (ibdr && jbdr && lbdr)
-   {
-      return 3;
-   }
-   int offset = 4;
-   if (jbdr && kbdr) // Edge DOF on j == 0 and k == 0
-   {
-      return offset + i - 1;
-   }
-   else if (kbdr && lbdr) // Edge DOF on k == 0 and l == 0
-   {
-      return offset + ref - 1 + j - 1;
-   }
-   else if (ibdr && kbdr) // Edge DOF on i == 0 and k == 0
-   {
-      return offset + 2 * (ref - 1) + ref - j - 1;
-   }
-   else if (ibdr && jbdr) // Edge DOF on i == 0 and j == 0
-   {
-      return offset + 3 * (ref - 1) + ref - k - 1;
-   }
-   else if (ibdr && lbdr) // Edge DOF on i == 0 and l == 0
-   {
-      return offset + 4 * (ref - 1) + ref - k - 1;
-   }
-   else if (jbdr && lbdr) // Edge DOF on j == 0 and l == 0
-   {
-      return offset + 5 * (ref - 1) + ref - k - 1;
-   }
-
-   // Recursive numbering for the faces
-   offset += 6 * (ref - 1);
-   if (kbdr)
-   {
-      int b_out[3];
-      b_out[0] = j-1;
-      b_out[1] = i-1;
-      b_out[2] = ref - i - j - 1;
-      return offset + BarycentricToVTKTriangle(b_out, ref-3);
-   }
-   else if (jbdr)
-   {
-      int b_out[3];
-      b_out[0] = i-1;
-      b_out[1] = k-1;
-      b_out[2] = ref - i - k - 1;
-      offset += (ref - 1) * (ref - 2) / 2;
-      return offset + BarycentricToVTKTriangle(b_out, ref-3);
-   }
-   else if (ibdr)
-   {
-      int b_out[3];
-      b_out[0] = k-1;
-      b_out[1] = j-1;
-      b_out[2] = ref - j - k - 1;
-      offset += (ref - 1) * (ref - 2);
-      return offset + BarycentricToVTKTriangle(b_out, ref-3);
-   }
-   else if (lbdr)
-   {
-      int b_out[3];
-      b_out[0] = ref-j-k-1;
-      b_out[1] = j-1;
-      b_out[2] = k-1;
-      offset += 3 * (ref - 1) * (ref - 2) / 2;
-      return offset + BarycentricToVTKTriangle(b_out, ref-3);
-   }
-
-   // Recursive numbering for interior
-   {
-      int b_out[4];
-      b_out[0] = i-1;
-      b_out[1] = j-1;
-      b_out[2] = k-1;
-      b_out[3] = ref - i - j - k - 1;
-      offset += 2 * (ref - 1) * (ref - 2);
-      return offset + BarycentricToGmshTet(b_out, ref-4);
-   }
-}
-
-int CartesianToGmshQuad(int idx_in[], int ref)
-{
-   int i = idx_in[0];
-   int j = idx_in[1];
-   // Do we lie on any of the edges
-   bool ibdr = (i == 0 || i == ref);
-   bool jbdr = (j == 0 || j == ref);
-   if (ibdr && jbdr) // Vertex DOF
-   {
-      return (i ? (j ? 2 : 1) : (j ? 3 : 0));
-   }
-   int offset = 4;
-   if (jbdr) // Edge DOF on j==0 or j==ref
-   {
-      return offset + (j ? 3*ref - 3 - i : i - 1);
-   }
-   else if (ibdr) // Edge DOF on i==0 or i==ref
-   {
-      return offset + (i ? ref - 1 + j - 1 : 4*ref - 4 - j);
-   }
-   else // Recursive numbering for interior
-   {
-      int idx_out[2];
-      idx_out[0] = i-1;
-      idx_out[1] = j-1;
-      offset += 4 * (ref - 1);
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-}
-
-int CartesianToGmshHex(int idx_in[], int ref)
-{
-   int i = idx_in[0];
-   int j = idx_in[1];
-   int k = idx_in[2];
-   // Do we lie on any of the edges
-   bool ibdr = (i == 0 || i == ref);
-   bool jbdr = (j == 0 || j == ref);
-   bool kbdr = (k == 0 || k == ref);
-   if (ibdr && jbdr && kbdr) // Vertex DOF
-   {
-      return (i ? (j ? (k ? 6 : 2) : (k ? 5 : 1)) :
-              (j ? (k ? 7 : 3) : (k ? 4 : 0)));
-   }
-   int offset = 8;
-   if (jbdr && kbdr) // Edge DOF on x-directed edge
-   {
-      return offset + (j ? (k ? 12*ref-12-i: 6*ref-6-i) :
-                       (k ? 8*ref-9+i: i-1));
-   }
-   else if (ibdr && kbdr) // Edge DOF on y-directed edge
-   {
-      return offset + (k ? (i ? 10*ref-11+j: 9*ref-10+j) :
-                       (i ? 3*ref-4+j: ref-2+j));
-   }
-   else if (ibdr && jbdr) // Edge DOF on z-directed edge
-   {
-      return offset + (i ? (j ? 6*ref-7+k: 4*ref-5+k) :
-                       (j ? 7*ref-8+k: 2*ref-3+k));
-   }
-   else if (ibdr) // Face DOF on x-directed face
-   {
-      int idx_out[2];
-      idx_out[0] = i ? j-1 : k-1;
-      idx_out[1] = i ? k-1 : j-1;
-      offset += (12 + (i ? 3 : 2) * (ref - 1)) * (ref - 1);
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-   else if (jbdr) // Face DOF on y-directed face
-   {
-      int idx_out[2];
-      idx_out[0] = j ? ref-i-1 : i-1;
-      idx_out[1] = j ? k-1 : k-1;
-      offset += (12 + (j ? 4 : 1) * (ref - 1)) * (ref - 1);
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-   else if (kbdr) // Face DOF on z-directed face
-   {
-      int idx_out[2];
-      idx_out[0] = k ? i-1 : j-1;
-      idx_out[1] = k ? j-1 : i-1;
-      offset += (12 + (k ? 5 : 0) * (ref - 1)) * (ref - 1);
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-   else // Recursive numbering for interior
-   {
-      int idx_out[3];
-      idx_out[0] = i-1;
-      idx_out[1] = j-1;
-      idx_out[2] = k-1;
-
-      offset += (12 + 6 * (ref - 1)) * (ref - 1);
-      return offset + CartesianToGmshHex(idx_out, ref-2);
-   }
-}
-
-int WedgeToGmshPri(int idx_in[], int ref)
-{
-   int i = idx_in[0];
-   int j = idx_in[1];
-   int k = idx_in[2];
-   int l = ref - i -j;
-   bool ibdr = (i == 0);
-   bool jbdr = (j == 0);
-   bool kbdr = (k == 0 || k == ref);
-   bool lbdr = (l == 0);
-   if (ibdr && jbdr && kbdr)
-   {
-      return k ? 3 : 0;
-   }
-   else if (jbdr && lbdr && kbdr)
-   {
-      return k ? 4 : 1;
-   }
-   else if (ibdr && lbdr && kbdr)
-   {
-      return k ? 5 : 2;
-   }
-   int offset = 6;
-   if (jbdr && kbdr)
-   {
-      return offset + (k ? 6 * (ref - 1) + i - 1: i - 1);
-   }
-   else if (ibdr && kbdr)
-   {
-      return offset + (k ? 7 * (ref -1) + j-1 : ref - 1 + j - 1);
-   }
-   else if (ibdr && jbdr)
-   {
-      return offset + 2 * (ref - 1) + k - 1;
-   }
-   else if (lbdr && kbdr)
-   {
-      return offset + (k ? 8 * (ref -1) + j - 1 : 3 * (ref - 1) + j - 1);
-   }
-   else if (jbdr && lbdr)
-   {
-      return offset + 4 * (ref - 1) + k - 1;
-   }
-   else if (ibdr && lbdr)
-   {
-      return offset + 5 * (ref - 1) + k - 1;
-   }
-   offset += 9 * (ref-1);
-   if (kbdr) // Triangular faces at k=0 and k=ref
-   {
-      int b_out[3];
-      b_out[0] = k ? i-1 : j-1;
-      b_out[1] = k ? j-1 : i-1;
-      b_out[2] = ref - i - j - 1;
-      offset += k ? (ref-1)*(ref-2) / 2: 0;
-      return offset + BarycentricToVTKTriangle(b_out, ref-3);
-   }
-   offset += (ref-1)*(ref-2);
-   if (jbdr) // Quadrilateral face at j=0
-   {
-      int idx_out[2];
-      idx_out[0] = i-1;
-      idx_out[1] = k-1;
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-   else if (ibdr) // Quadrilateral face at i=0
-   {
-      int idx_out[2];
-      idx_out[0] = k-1;
-      idx_out[1] = j-1;
-      offset += (ref-1)*(ref-1);
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-   else if (lbdr) // Quadrilateral face at l=ref-i-j=0
-   {
-      int idx_out[2];
-      idx_out[0] = j-1;
-      idx_out[1] = k-1;
-      offset += 2*(ref-1)*(ref-1);
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-   offset += 3*(ref-1)*(ref-1);
-   // The Gmsh Prism interiors are a tensor product of segments of order ref-2
-   // and triangles of order ref-3
-   {
-      int b_out[3];
-      b_out[0] = i-1;
-      b_out[1] = j-1;
-      b_out[2] = ref - i - j - 1;
-      int ot = BarycentricToVTKTriangle(b_out, ref-3);
-      int os = (k==1) ? 0 : (k == ref-1 ? 1 : k);
-      return offset + (ref-1) * ot + os;
-   }
-}
-
-int CartesianToGmshPyramid(int idx_in[], int ref)
-{
-   int i = idx_in[0];
-   int j = idx_in[1];
-   int k = idx_in[2];
-   // Do we lie on any of the edges
-   bool ibdr = (i == 0 || i == ref-k);
-   bool jbdr = (j == 0 || j == ref-k);
-   bool kbdr = (k == 0);
-   if (ibdr && jbdr && kbdr)
-   {
-      return i ? (j ? 2 : 1): (j ? 3 : 0);
-   }
-   else if (k == ref)
-   {
-      return 4;
-   }
-   int offset = 5;
-   if (jbdr && kbdr)
-   {
-      return offset + (j ? (6 * ref - 6 - i) : (i - 1));
-   }
-   else if (ibdr && kbdr)
-   {
-      return offset + (i ? (3 * ref - 4 + j) : (ref - 2 + j));
-   }
-   else if (ibdr && jbdr)
-   {
-      return offset + (i ? (j ? 6 : 4) : (j ? 7 : 2 )) * (ref-1) + k - 1;
-   }
-   offset += 8*(ref-1);
-   if (jbdr)
-   {
-      int b_out[3];
-      b_out[0] = j ? ref - i - k - 1 : i - 1;
-      b_out[1] = k - 1;
-      b_out[2] = (j ? i - 1 : ref - i - k - 1);
-      offset += (j ? 3 : 0) * (ref - 1) * (ref - 2) / 2;
-      return offset + BarycentricToVTKTriangle(b_out, ref-3);
-   }
-   else if (ibdr)
-   {
-      int b_out[3];
-      b_out[0] = i ? j - 1: ref - j - k - 1;
-      b_out[1] = k - 1;
-      b_out[2] = (i ? ref - j - k - 1: j - 1);
-      offset += (i ? 2 : 1) * (ref - 1) * (ref - 2) / 2;
-      return offset + BarycentricToVTKTriangle(b_out, ref-3);
-   }
-   else if (kbdr)
-   {
-      int idx_out[2];
-      idx_out[0] = k ? i-1 : j-1;
-      idx_out[1] = k ? j-1 : i-1;
-      offset += 2 * (ref - 1) * (ref - 2);
-      return offset + CartesianToGmshQuad(idx_out, ref-2);
-   }
-   offset += (2 * (ref - 2) + (ref - 1)) * (ref - 1) ;
-   {
-      int idx_out[3];
-      idx_out[0] = i-1;
-      idx_out[1] = j-1;
-      idx_out[2] = k-1;
-      return offset + CartesianToGmshPyramid(idx_out, ref-3);
-   }
-}
-
-void GmshHOSegmentMapping(int order, int *map)
-{
-   map[0] = 0;
-   map[order] = 1;
-   for (int i=1; i<order; i++)
-   {
-      map[i] = i + 1;
-   }
-}
-
-void GmshHOTriangleMapping(int order, int *map)
-{
-   int b[3];
-   int o = 0;
-   for (b[1]=0; b[1]<=order; ++b[1])
-   {
-      for (b[0]=0; b[0]<=order-b[1]; ++b[0])
-      {
-         b[2] = order - b[0] - b[1];
-         map[o] = BarycentricToVTKTriangle(b, order);
-         o++;
-      }
-   }
-}
-
-void GmshHOQuadrilateralMapping(int order, int *map)
-{
-   int b[2];
-   int o = 0;
-   for (b[1]=0; b[1]<=order; b[1]++)
-   {
-      for (b[0]=0; b[0]<=order; b[0]++)
-      {
-         map[o] = CartesianToGmshQuad(b, order);
-         o++;
-      }
-   }
-}
-
-void GmshHOTetrahedronMapping(int order, int *map)
-{
-   int b[4];
-   int o = 0;
-   for (b[2]=0; b[2]<=order; ++b[2])
-   {
-
-      for (b[1]=0; b[1]<=order-b[2]; ++b[1])
-      {
-         for (b[0]=0; b[0]<=order-b[1]-b[2]; ++b[0])
-         {
-            b[3] = order - b[0] - b[1] - b[2];
-            map[o] = BarycentricToGmshTet(b, order);
-            o++;
-         }
-      }
-   }
-}
-
-void GmshHOHexahedronMapping(int order, int *map)
-{
-   int b[3];
-   int o = 0;
-   for (b[2]=0; b[2]<=order; b[2]++)
-   {
-      for (b[1]=0; b[1]<=order; b[1]++)
-      {
-         for (b[0]=0; b[0]<=order; b[0]++)
-         {
-            map[o] = CartesianToGmshHex(b, order);
-            o++;
-         }
-      }
-   }
-}
-
-void GmshHOWedgeMapping(int order, int *map)
-{
-   int b[3];
-   int o = 0;
-   for (b[2]=0; b[2]<=order; b[2]++)
-   {
-      for (b[1]=0; b[1]<=order; b[1]++)
-      {
-         for (b[0]=0; b[0]<=order - b[1]; b[0]++)
-         {
-            map[o] = WedgeToGmshPri(b, order);
-            o++;
-         }
-      }
-   }
-}
-
-void GmshHOPyramidMapping(int order, int *map)
-{
-   int b[3];
-   int o = 0;
-   for (b[2]=0; b[2]<=order; b[2]++)
-   {
-      for (b[1]=0; b[1]<=order - b[2]; b[1]++)
-      {
-         for (b[0]=0; b[0]<=order - b[2]; b[0]++)
-         {
-            map[o] = CartesianToGmshPyramid(b, order);
-            o++;
-         }
-      }
-   }
-}
-
-} // namespace mfem

mesh/gmsh.hpp

@@ -1,55 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_GMSH
-#define MFEM_GMSH
-
-namespace mfem
-{
-
-// Helpers for reading high order elements in Gmsh format
-
-/** @name Gmsh High-Order Vertex Mappings
-
-    These functions generate the mappings needed to translate the order of
-    Gmsh's high-order vertices into MFEM's L2 degree of freedom ordering. The
-    mapping is defined so that MFEM_DoF[i] = Gmsh_Vert[map[i]]. The @a map
-    array must already be allocated with the proper number of entries for the
-    element type at the given element @a order.
-*/
-///@{
-
-/// @brief Generate Gmsh vertex mapping for a Segment
-void GmshHOSegmentMapping(int order, int *map);
-
-/// @brief Generate Gmsh vertex mapping for a Triangle
-void GmshHOTriangleMapping(int order, int *map);
-
-/// @brief Generate Gmsh vertex mapping for a Quadrilateral
-void GmshHOQuadrilateralMapping(int order, int *map);
-
-/// @brief Generate Gmsh vertex mapping for a Tetrahedron
-void GmshHOTetrahedronMapping(int order, int *map);
-
-/// @brief Generate Gmsh vertex mapping for a Hexahedron
-void GmshHOHexahedronMapping(int order, int *map);
-
-/// @brief Generate Gmsh vertex mapping for a Wedge
-void GmshHOWedgeMapping(int order, int *map);
-
-/// @brief Generate Gmsh vertex mapping for a Pyramid
-void GmshHOPyramidMapping(int order, int *map);
-
-///@}
-
-} // namespace mfem
-
-#endif

mesh/mesh.cpp

@@ -338,7 +338,6 @@ void Mesh::GetElementTransformation(int i, IsoparametricTransformation *ElTr)
    ElTr->Attribute = GetAttribute(i);
    ElTr->ElementNo = i;
    ElTr->ElementType = ElementTransformation::ELEMENT;
-   ElTr->Reset();
    if (Nodes == NULL)
    {
       GetPointMatrix(i, ElTr->GetPointMat());
@@ -371,7 +370,6 @@ void Mesh::GetElementTransformation(int i, const Vector &nodes,
    ElTr->ElementNo = i;
    ElTr->ElementType = ElementTransformation::ELEMENT;
    DenseMatrix &pm = ElTr->GetPointMat();
-   ElTr->Reset();
    nodes.HostRead();
    if (Nodes == NULL)
    {
@@ -426,7 +424,6 @@ void Mesh::GetBdrElementTransformation(int i, IsoparametricTransformation* ElTr)
    ElTr->ElementNo = i; // boundary element number
    ElTr->ElementType = ElementTransformation::BDR_ELEMENT;
    DenseMatrix &pm = ElTr->GetPointMat();
-   ElTr->Reset();
    if (Nodes == NULL)
    {
       GetBdrPointMatrix(i, pm);
@@ -483,7 +480,6 @@ void Mesh::GetFaceTransformation(int FaceNo, IsoparametricTransformation *FTr)
    FTr->ElementNo = FaceNo;
    FTr->ElementType = ElementTransformation::FACE;
    DenseMatrix &pm = FTr->GetPointMat();
-   FTr->Reset();
    if (Nodes == NULL)
    {
       const int *v = (Dim == 1) ? &FaceNo : faces[FaceNo]->GetVertices();
@@ -566,7 +562,6 @@ void Mesh::GetEdgeTransformation(int EdgeNo, IsoparametricTransformation *EdTr)
    EdTr->ElementNo = EdgeNo;
    EdTr->ElementType = ElementTransformation::EDGE;
    DenseMatrix &pm = EdTr->GetPointMat();
-   EdTr->Reset();
    if (Nodes == NULL)
    {
       Array<int> v;
@@ -619,7 +614,6 @@ void Mesh::GetLocalPtToSegTransformation(
 {
    const IntegrationRule *SegVert;
    DenseMatrix &locpm = Transf.GetPointMat();
-   Transf.Reset();
 
    Transf.SetFE(&PointFE);
    SegVert = Geometries.GetVertices(Geometry::SEGMENT);
@@ -635,7 +629,6 @@ void Mesh::GetLocalSegToTriTransformation(
    const int *tv, *so;
    const IntegrationRule *TriVert;
    DenseMatrix &locpm = Transf.GetPointMat();
-   Transf.Reset();
 
    Transf.SetFE(&SegmentFE);
    tv = tri_t::Edges[i/64];  //  (i/64) is the local face no. in the triangle
@@ -655,7 +648,6 @@ void Mesh::GetLocalSegToQuadTransformation(
    const int *qv, *so;
    const IntegrationRule *QuadVert;
    DenseMatrix &locpm = Transf.GetPointMat();
-   Transf.Reset();
 
    Transf.SetFE(&SegmentFE);
    qv = quad_t::Edges[i/64]; //  (i/64) is the local face no. in the quad
@@ -673,7 +665,6 @@ void Mesh::GetLocalTriToTetTransformation(
    IsoparametricTransformation &Transf, int i)
 {
    DenseMatrix &locpm = Transf.GetPointMat();
-   Transf.Reset();
 
    Transf.SetFE(&TriangleFE);
    //  (i/64) is the local face no. in the tet
@@ -697,7 +688,6 @@ void Mesh::GetLocalTriToWdgTransformation(
    IsoparametricTransformation &Transf, int i)
 {
    DenseMatrix &locpm = Transf.GetPointMat();
-   Transf.Reset();
 
    Transf.SetFE(&TriangleFE);
    //  (i/64) is the local face no. in the pri
@@ -723,7 +713,6 @@ void Mesh::GetLocalQuadToHexTransformation(
    IsoparametricTransformation &Transf, int i)
 {
    DenseMatrix &locpm = Transf.GetPointMat();
-   Transf.Reset();
 
    Transf.SetFE(&QuadrilateralFE);
    //  (i/64) is the local face no. in the hex
@@ -745,7 +734,6 @@ void Mesh::GetLocalQuadToWdgTransformation(
    IsoparametricTransformation &Transf, int i)
 {
    DenseMatrix &locpm = Transf.GetPointMat();
-   Transf.Reset();
 
    Transf.SetFE(&QuadrilateralFE);
    //  (i/64) is the local face no. in the pri

mesh/pmesh.cpp

@@ -1691,7 +1691,6 @@ void ParMesh::GetFaceNbrElementTransformation(
    ElTr->Attribute = elem->GetAttribute();
    ElTr->ElementNo = NumOfElements + i;
    ElTr->ElementType = ElementTransformation::ELEMENT;
-   ElTr->Reset();
 
    if (Nodes == NULL)
    {
@@ -2371,7 +2370,6 @@ void ParMesh::GetGhostFaceTransformation(
 {
    // calculate composition of FETr->Loc1 and FETr->Elem1
    DenseMatrix &face_pm = FETr->GetPointMat();
-   FETr->Reset();
    if (Nodes == NULL)
    {
       FETr->Elem1->Transform(FETr->Loc1.Transf.GetPointMat(), face_pm);

mesh/vtk.hpp

@@ -37,8 +37,6 @@ void CreateVTKElementConnectivity(Array<int> &con, Geometry::Type geom,
 void WriteVTKEncodedCompressed(std::ostream &out, const void *bytes,
                                uint32_t nbytes, int compression_level);
 
-int BarycentricToVTKTriangle(int *b, int ref);
-
 const char *VTKByteOrder();
 
 } // namespace mfem

miniapps/meshing/mesh-explorer.cpp

@@ -39,7 +39,7 @@
 using namespace mfem;
 using namespace std;
 
-// This transformation can be applied to a mesh with the 't' menu option.
+// This tranformation can be applied to a mesh with the 't' menu option.
 void transformation(const Vector &p, Vector &v)
 {
    // simple shear transformation
@@ -72,28 +72,6 @@ double region(const Vector &p)
    return std::max(std::max(x - 0.25, -y), y - 1.0);
 }
 
-// The projection of this function can be plotted with the 'l' menu option
-double f(const Vector &p)
-{
-   double x = p(0);
-   double y = p.Size() > 1 ? p(1) : 0.0;
-   double z = p.Size() > 2 ? p(2) : 0.0;
-
-   if (1)
-   {
-      // torus in the xy-plane
-      const double r_big = 2.0;
-      const double r_small = 1.0;
-      return hypot(r_big - hypot(x, y), z) - r_small;
-   }
-   if (0)
-   {
-      // sphere at the origin:
-      const double r = 1.0;
-      return hypot(hypot(x, y), z) - r;
-   }
-}
-
 Mesh *read_par_mesh(int np, const char *mesh_prefix)
 {
    Mesh *mesh;
@@ -351,7 +329,6 @@ int main (int argc, char *argv[])
            "e) View elements\n"
            "h) View element sizes, h\n"
            "k) View element ratios, kappa\n"
-           "l) Plot a function\n"
            "x) Print sub-element stats\n"
            "f) Find physical point in reference space\n"
            "p) Generate a partitioning\n"
@@ -690,7 +667,7 @@ int main (int argc, char *argv[])
          }
       }
 
-      // These are most of the cases that open a new GLVis window
+      // These are the cases that open a new GLVis window
       if (mk == 'm' || mk == 'b' || mk == 'e' || mk == 'v' || mk == 'h' ||
           mk == 'k' || mk == 'p')
       {
@@ -1003,43 +980,6 @@ int main (int argc, char *argv[])
          delete bdr_attr_fespace;
       }
 
-      if (mk == 'l')
-      {
-         // Project and plot the function 'f'
-         int p;
-         FiniteElementCollection *fec = NULL;
-         cout << "Enter projection space order: " << flush;
-         cin >> p;
-         if (p >= 1)
-         {
-            fec = new H1_FECollection(p, mesh->Dimension(),
-                                      BasisType::GaussLobatto);
-         }
-         else
-         {
-            fec = new DG_FECollection(-p, mesh->Dimension(),
-                                      BasisType::GaussLegendre);
-         }
-         FiniteElementSpace fes(mesh, fec);
-         GridFunction level(&fes);
-         FunctionCoefficient coeff(f);
-         level.ProjectCoefficient(coeff);
-         char vishost[] = "localhost";
-         int  visport   = 19916;
-         socketstream sol_sock(vishost, visport);
-         if (sol_sock.is_open())
-         {
-            sol_sock.precision(14);
-            sol_sock << "solution\n" << *mesh << level << flush;
-         }
-         else
-         {
-            cout << "Unable to connect to "
-                 << vishost << ':' << visport << endl;
-         }
-         delete fec;
-      }
-
       if (mk == 'S')
       {
          const char mesh_file[] = "mesh-explorer.mesh";

miniapps/meshing/mesh-optimizer.cpp

@@ -41,8 +41,7 @@
 //
 //   Adapted discrete size:
 //     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 5 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
-//   Adapted discrete size; explicit combo of metrics; mixed tri/quad mesh:
-//     mesh-optimizer -m ../../data/square-mixed.mesh -o 2 -rs 2 -mid 2 -tid 5 -ni 200 -bnd -qo 6 -cmb 2 -nor
+//     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 2 -tid 5 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8 -cmb 2 -nor
 //   Adapted discrete size+aspect_ratio:
 //     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 6 -ni 100
 //     mesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 6 -ni 100 -qo 6 -ex -st 1 -nor
@@ -76,8 +75,6 @@
 //     mesh-optimizer -o 3 -rs 0 -mid 1 -tid 1 -ni 100 -ls 2 -li 100 -bnd -qt 1 -qo 8 -lc 10
 //   ICF combo shape + size (rings, slow convergence):
 //     mesh-optimizer -o 3 -rs 0 -mid 1 -tid 1 -ni 1000 -ls 2 -li 100 -bnd -qt 1 -qo 8 -cmb 1
-//   Mixed tet / cube / hex mesh with limiting:
-//     mesh-optimizer -m ../../data/fichera-mixed-p2.mesh -o 4 -rs 1 -mid 301 -tid 1 -fix-bnd -qo 6 -nor -lc 0.25
 //   3D pinched sphere shape (the mesh is in the mfem/data GitHub repository):
 //   * mesh-optimizer -m ../../../mfem_data/ball-pert.mesh -o 4 -rs 0 -mid 303 -tid 1 -ni 20 -ls 2 -li 500 -fix-bnd
 //   2D non-conforming shape and equal size:
@@ -549,36 +546,23 @@ int main(int argc, char *argv[])
    if (fdscheme) { he_nlf_integ->EnableFiniteDifferences(x); }
    he_nlf_integ->SetExactActionFlag(exactaction);
 
-   // Setup the quadrature rules for the TMOP integrator.
-   IntegrationRules *irules = NULL;
+   // 12. Setup the quadrature rule for the non-linear form integrator.
+   const IntegrationRule *ir = NULL;
+   const int geom_type = fespace->GetFE(0)->GetGeomType();
    switch (quad_type)
    {
-      case 1: irules = &IntRulesLo; break;
-      case 2: irules = &IntRules; break;
-      case 3: irules = &IntRulesCU; break;
-      default: cout << "Unknown quad_type: " << quad_type << endl; return 3;
-   }
-   he_nlf_integ->SetIntegrationRules(*irules, quad_order);
-   if (dim == 2)
-   {
-      cout << "Triangle quadrature points: "
-           << irules->Get(Geometry::TRIANGLE, quad_order).GetNPoints()
-           << "\nQuadrilateral quadrature points: "
-           << irules->Get(Geometry::SQUARE, quad_order).GetNPoints() << endl;
-   }
-   if (dim == 3)
-   {
-      cout << "Tetrahedron quadrature points: "
-           << irules->Get(Geometry::TETRAHEDRON, quad_order).GetNPoints()
-           << "\nHexahedron quadrature points: "
-           << irules->Get(Geometry::CUBE, quad_order).GetNPoints()
-           << "\nPrism quadrature points: "
-           << irules->Get(Geometry::PRISM, quad_order).GetNPoints() << endl;
+      case 1: ir = &IntRulesLo.Get(geom_type, quad_order); break;
+      case 2: ir = &IntRules.Get(geom_type, quad_order); break;
+      case 3: ir = &IntRulesCU.Get(geom_type, quad_order); break;
+      default: cout << "Unknown quad_type: " << quad_type << endl;
+         delete he_nlf_integ; return 3;
    }
+   cout << "Quadrature points per cell: " << ir->GetNPoints() << endl;
+   he_nlf_integ->SetIntegrationRule(*ir);
 
    if (normalization) { he_nlf_integ->EnableNormalization(x0); }
 
-   // Limit the node movement.
+   // 13. Limit the node movement.
    // The limiting distances can be given by a general function of space.
    GridFunction dist(fespace);
    dist = 1.0;
@@ -616,7 +600,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 12. Setup the final NonlinearForm (which defines the integral of interest,
+   // 14. Setup the final NonlinearForm (which defines the integral of interest,
    //     its first and second derivatives). Here we can use a combination of
    //     metrics, i.e., optimize the sum of two integrals, where both are
    //     scaled by used-defined space-dependent weights. Note that there are no
@@ -628,7 +612,6 @@ int main(int argc, char *argv[])
    TargetConstructor *target_c2 = NULL;
    FunctionCoefficient coeff2(weight_fun);
 
-   // Explicit combination of metrics.
    if (combomet > 0)
    {
       // First metric.
@@ -648,7 +631,7 @@ int main(int argc, char *argv[])
          he_nlf_integ2->SetCoefficient(coeff2);
       }
       else { he_nlf_integ2 = new TMOP_Integrator(metric2, target_c); }
-      he_nlf_integ2->SetIntegrationRules(*irules, quad_order);
+      he_nlf_integ2->SetIntegrationRule(*ir);
       if (fdscheme) { he_nlf_integ2->EnableFiniteDifferences(x); }
       he_nlf_integ2->SetExactActionFlag(exactaction);
 
@@ -664,15 +647,14 @@ int main(int argc, char *argv[])
 
    const double init_energy = a.GetGridFunctionEnergy(x);
 
-   // Visualize the starting mesh and metric values.
-   // Note that for combinations of metrics, this only shows the first metric.
+   // 15. Visualize the starting mesh and metric values.
    if (visualization)
    {
       char title[] = "Initial metric values";
       vis_tmop_metric_s(mesh_poly_deg, *metric, *target_c, *mesh, title, 0);
    }
 
-   // 13. Fix all boundary nodes, or fix only a given component depending on the
+   // 16. Fix all boundary nodes, or fix only a given component depending on the
    //     boundary attributes of the given mesh. Attributes 1/2/3 correspond to
    //     fixed x/y/z components of the node. Attribute 4 corresponds to an
    //     entirely fixed node. Other boundary attributes do not affect the node
@@ -685,10 +667,10 @@ int main(int argc, char *argv[])
    }
    else
    {
+      const int nd  = fespace->GetBE(0)->GetDof();
       int n = 0;
       for (int i = 0; i < mesh->GetNBE(); i++)
       {
-         const int nd = fespace->GetBE(i)->GetDof();
          const int attr = mesh->GetBdrElement(i)->GetAttribute();
          MFEM_VERIFY(!(dim == 2 && attr == 3),
                      "Boundary attribute 3 must be used only for 3D meshes. "
@@ -701,7 +683,6 @@ int main(int argc, char *argv[])
       n = 0;
       for (int i = 0; i < mesh->GetNBE(); i++)
       {
-         const int nd = fespace->GetBE(i)->GetDof();
          const int attr = mesh->GetBdrElement(i)->GetAttribute();
          fespace->GetBdrElementVDofs(i, vdofs);
          if (attr == 1) // Fix x components.
@@ -728,7 +709,7 @@ int main(int argc, char *argv[])
       a.SetEssentialVDofs(ess_vdofs);
    }
 
-   // 14. As we use the Newton method to solve the resulting nonlinear system,
+   // 17. As we use the Newton method to solve the resulting nonlinear system,
    //     here we setup the linear solver for the system's Jacobian.
    Solver *S = NULL;
    const double linsol_rtol = 1e-12;
@@ -755,17 +736,15 @@ int main(int argc, char *argv[])
       S = minres;
    }
 
-   // Compute the minimum det(J) of the starting mesh.
+   // 18. Compute the minimum det(J) of the starting mesh.
    tauval = infinity();
    const int NE = mesh->GetNE();
    for (int i = 0; i < NE; i++)
    {
-      const IntegrationRule &ir =
-         irules->Get(fespace->GetFE(i)->GetGeomType(), quad_order);
       ElementTransformation *transf = mesh->GetElementTransformation(i);
-      for (int j = 0; j < ir.GetNPoints(); j++)
+      for (int j = 0; j < ir->GetNPoints(); j++)
       {
-         transf->SetIntPoint(&ir.IntPoint(j));
+         transf->SetIntPoint(&ir->IntPoint(j));
          tauval = min(tauval, transf->Jacobian().Det());
       }
    }
@@ -773,11 +752,7 @@ int main(int argc, char *argv[])
    tauval -= 0.01 * h0.Min(); // Slightly below minJ0 to avoid div by 0.
 
    // Perform the nonlinear optimization.
-   const IntegrationRule &ir =
-      irules->Get(fespace->GetFE(0)->GetGeomType(), quad_order);
-   TMOPNewtonSolver solver(ir, solver_type);
-   // Provide all integration rules in case of a mixed mesh.
-   solver.SetIntegrationRules(*irules, quad_order);
+   TMOPNewtonSolver solver(*ir, solver_type);
    if (solver_type == 0)
    {
       // Specify linear solver when we use a Newton-based solver.
@@ -795,7 +770,7 @@ int main(int argc, char *argv[])
       cout << "Nonlinear solver: rtol = " << solver_rtol << " not achieved.\n";
    }
 
-   // 15. Save the optimized mesh to a file. This output can be viewed later
+   // 20. Save the optimized mesh to a file. This output can be viewed later
    //     using GLVis: "glvis -m optimized.mesh".
    {
       ofstream mesh_ofs("optimized.mesh");
@@ -803,7 +778,7 @@ int main(int argc, char *argv[])
       mesh->Print(mesh_ofs);
    }
 
-   // 16. Compute the amount of energy decrease.
+   // 21. Compute the amount of energy decrease.
    const double fin_energy = a.GetGridFunctionEnergy(x);
    double metric_part = fin_energy;
    if (lim_const > 0.0 || adapt_lim_const > 0.0)
@@ -823,7 +798,7 @@ int main(int argc, char *argv[])
    cout << "The strain energy decreased by: " << setprecision(12)
         << (init_energy - fin_energy) * 100.0 / init_energy << " %." << endl;
 
-   // 17. Visualize the final mesh and metric values.
+   // 22. Visualize the final mesh and metric values.
    if (visualization)
    {
       char title[] = "Final metric values";
@@ -837,7 +812,7 @@ int main(int argc, char *argv[])
                              600, 600, 300, 300);
    }
 
-   // 18. Visualize the mesh displacement.
+   // 23. Visualize the mesh displacement.
    if (visualization)
    {
       x0 -= x;
@@ -852,7 +827,7 @@ int main(int argc, char *argv[])
            << "keys jRmclA" << endl;
    }
 
-   // 19. Free the used memory.
+   // 24. Free the used memory.
    delete S;
    delete target_c2;
    delete metric2;

miniapps/meshing/pmesh-optimizer.cpp

@@ -41,8 +41,7 @@
 //
 //   Adapted discrete size:
 //     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 5 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8
-//   Adapted discrete size; explicit combo of metrics; mixed tri/quad mesh:
-//     mpirun -np 4 pmesh-optimizer -m ../../data/square-mixed.mesh -o 2 -rs 2 -mid 2 -tid 5 -ni 200 -bnd -qo 6 -cmb 2 -nor
+//     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 2 -tid 5 -ni 200 -ls 2 -li 100 -bnd -qt 1 -qo 8 -cmb 2 -nor
 //   Adapted discrete size+aspect_ratio:
 //     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 6 -ni 100
 //     mpirun -np 4 pmesh-optimizer -m square01.mesh -o 2 -rs 2 -mid 7 -tid 6 -ni 100 -qo 6 -ex -st 1 -nor
@@ -76,8 +75,6 @@
 //     mpirun -np 4 pmesh-optimizer -o 3 -rs 0 -mid 1 -tid 1 -ni 100 -ls 2 -li 100 -bnd -qt 1 -qo 8 -lc 10
 //   ICF combo shape + size (rings, slow convergence):
 //     mpirun -np 4 pmesh-optimizer -o 3 -rs 0 -mid 1 -tid 1 -ni 1000 -ls 2 -li 100 -bnd -qt 1 -qo 8 -cmb 1
-//   Mixed tet / cube / hex mesh with limiting:
-//     mpirun -np 4 pmesh-optimizer -m ../../data/fichera-mixed-p2.mesh -o 4 -rs 1 -mid 301 -tid 1 -fix-bnd -qo 6 -nor -lc 0.25
 //   3D pinched sphere shape (the mesh is in the mfem/data GitHub repository):
 //   * mpirun -np 4 pmesh-optimizer -m ../../../mfem_data/ball-pert.mesh -o 4 -rs 0 -mid 303 -tid 1 -ni 20 -ls 2 -li 500 -fix-bnd
 //   2D non-conforming shape and equal size:
@@ -485,10 +482,11 @@ int main (int argc, char *argv[])
             }
          }
          double volume_all, volume_ind_all;
+         int NE_ALL;
          MPI_Allreduce(&volume, &volume_all, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
          MPI_Allreduce(&volume_ind, &volume_ind_all, 1, MPI_DOUBLE, MPI_SUM,
                        MPI_COMM_WORLD);
-         const int NE_ALL = pmesh->GetGlobalNE();
+         MPI_Allreduce(&NE, &NE_ALL, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
 
          const double avg_zone_size = volume_all / NE_ALL;
 
@@ -588,38 +586,25 @@ int main (int argc, char *argv[])
    if (fdscheme) { he_nlf_integ->EnableFiniteDifferences(x); }
    he_nlf_integ->SetExactActionFlag(exactaction);
 
-   // Setup the quadrature rules for the TMOP integrator.
-   IntegrationRules *irules = NULL;
+   // 13. Setup the quadrature rule for the non-linear form integrator.
+   const IntegrationRule *ir = NULL;
+   const int geom_type = pfespace->GetFE(0)->GetGeomType();
    switch (quad_type)
    {
-      case 1: irules = &IntRulesLo; break;
-      case 2: irules = &IntRules; break;
-      case 3: irules = &IntRulesCU; break;
+      case 1: ir = &IntRulesLo.Get(geom_type, quad_order); break;
+      case 2: ir = &IntRules.Get(geom_type, quad_order); break;
+      case 3: ir = &IntRulesCU.Get(geom_type, quad_order); break;
       default:
          if (myid == 0) { cout << "Unknown quad_type: " << quad_type << endl; }
          return 3;
    }
-   he_nlf_integ->SetIntegrationRules(*irules, quad_order);
-   if (myid == 0 && dim == 2)
-   {
-      cout << "Triangle quadrature points: "
-           << irules->Get(Geometry::TRIANGLE, quad_order).GetNPoints()
-           << "\nQuadrilateral quadrature points: "
-           << irules->Get(Geometry::SQUARE, quad_order).GetNPoints() << endl;
-   }
-   if (myid == 0 && dim == 3)
-   {
-      cout << "Tetrahedron quadrature points: "
-           << irules->Get(Geometry::TETRAHEDRON, quad_order).GetNPoints()
-           << "\nHexahedron quadrature points: "
-           << irules->Get(Geometry::CUBE, quad_order).GetNPoints()
-           << "\nPrism quadrature points: "
-           << irules->Get(Geometry::PRISM, quad_order).GetNPoints() << endl;
-   }
+   if (myid == 0)
+   { cout << "Quadrature points per cell: " << ir->GetNPoints() << endl; }
+   he_nlf_integ->SetIntegrationRule(*ir);
 
    if (normalization) { he_nlf_integ->ParEnableNormalization(x0); }
 
-   // Limit the node movement.
+   // 14. Limit the node movement.
    // The limiting distances can be given by a general function of space.
    ParGridFunction dist(pfespace);
    dist = 1.0;
@@ -657,7 +642,7 @@ int main (int argc, char *argv[])
       }
    }
 
-   // 13. Setup the final NonlinearForm (which defines the integral of interest,
+   // 15. Setup the final NonlinearForm (which defines the integral of interest,
    //     its first and second derivatives). Here we can use a combination of
    //     metrics, i.e., optimize the sum of two integrals, where both are
    //     scaled by used-defined space-dependent weights.  Note that there are
@@ -669,7 +654,6 @@ int main (int argc, char *argv[])
    TargetConstructor *target_c2 = NULL;
    FunctionCoefficient coeff2(weight_fun);
 
-   // Explicit combination of metrics.
    if (combomet > 0)
    {
       // First metric.
@@ -689,7 +673,7 @@ int main (int argc, char *argv[])
          he_nlf_integ2->SetCoefficient(coeff2);
       }
       else { he_nlf_integ2 = new TMOP_Integrator(metric2, target_c); }
-      he_nlf_integ2->SetIntegrationRules(*irules, quad_order);
+      he_nlf_integ2->SetIntegrationRule(*ir);
       if (fdscheme) { he_nlf_integ2->EnableFiniteDifferences(x); }
       he_nlf_integ2->SetExactActionFlag(exactaction);
 
@@ -705,15 +689,14 @@ int main (int argc, char *argv[])
 
    const double init_energy = a.GetParGridFunctionEnergy(x);
 
-   // Visualize the starting mesh and metric values.
-   // Note that for combinations of metrics, this only shows the first metric.
+   // 16. Visualize the starting mesh and metric values.
    if (visualization)
    {
       char title[] = "Initial metric values";
       vis_tmop_metric_p(mesh_poly_deg, *metric, *target_c, *pmesh, title, 0);
    }
 
-   // 14. Fix all boundary nodes, or fix only a given component depending on the
+   // 17. Fix all boundary nodes, or fix only a given component depending on the
    //     boundary attributes of the given mesh.  Attributes 1/2/3 correspond to
    //     fixed x/y/z components of the node.  Attribute 4 corresponds to an
    //     entirely fixed node.  Other boundary attributes do not affect the node
@@ -726,10 +709,10 @@ int main (int argc, char *argv[])
    }
    else
    {
+      const int nd  = pfespace->GetBE(0)->GetDof();
       int n = 0;
       for (int i = 0; i < pmesh->GetNBE(); i++)
       {
-         const int nd = pfespace->GetBE(i)->GetDof();
          const int attr = pmesh->GetBdrElement(i)->GetAttribute();
          MFEM_VERIFY(!(dim == 2 && attr == 3),
                      "Boundary attribute 3 must be used only for 3D meshes. "
@@ -742,7 +725,6 @@ int main (int argc, char *argv[])
       n = 0;
       for (int i = 0; i < pmesh->GetNBE(); i++)
       {
-         const int nd = pfespace->GetBE(i)->GetDof();
          const int attr = pmesh->GetBdrElement(i)->GetAttribute();
          pfespace->GetBdrElementVDofs(i, vdofs);
          if (attr == 1) // Fix x components.
@@ -769,7 +751,7 @@ int main (int argc, char *argv[])
       a.SetEssentialVDofs(ess_vdofs);
    }
 
-   // 15. As we use the Newton method to solve the resulting nonlinear system,
+   // 18. As we use the Newton method to solve the resulting nonlinear system,
    //     here we setup the linear solver for the system's Jacobian.
    Solver *S = NULL;
    const double linsol_rtol = 1e-12;
@@ -796,17 +778,15 @@ int main (int argc, char *argv[])
       S = minres;
    }
 
-   // Compute the minimum det(J) of the starting mesh.
+   // 19. Compute the minimum det(J) of the starting mesh.
    tauval = infinity();
    const int NE = pmesh->GetNE();
    for (int i = 0; i < NE; i++)
    {
-      const IntegrationRule &ir =
-         irules->Get(pfespace->GetFE(i)->GetGeomType(), quad_order);
       ElementTransformation *transf = pmesh->GetElementTransformation(i);
-      for (int j = 0; j < ir.GetNPoints(); j++)
+      for (int j = 0; j < ir->GetNPoints(); j++)
       {
-         transf->SetIntPoint(&ir.IntPoint(j));
+         transf->SetIntPoint(&ir->IntPoint(j));
          tauval = min(tauval, transf->Jacobian().Det());
       }
    }
@@ -820,11 +800,7 @@ int main (int argc, char *argv[])
    tauval -= 0.01 * h0min_all; // Slightly below minJ0 to avoid div by 0.
 
    // Perform the nonlinear optimization.
-   const IntegrationRule &ir =
-      irules->Get(pfespace->GetFE(0)->GetGeomType(), quad_order);
-   TMOPNewtonSolver solver(pfespace->GetComm(), ir, solver_type);
-   // Provide all integration rules in case of a mixed mesh.
-   solver.SetIntegrationRules(*irules, quad_order);
+   TMOPNewtonSolver solver(pfespace->GetComm(), *ir, solver_type);
    if (solver_type == 0)
    {
       // Specify linear solver when we use a Newton-based solver.
@@ -842,7 +818,7 @@ int main (int argc, char *argv[])
       cout << "Nonlinear solver: rtol = " << solver_rtol << " not achieved.\n";
    }
 
-   // 16. Save the optimized mesh to a file. This output can be viewed later
+   // 21. Save the optimized mesh to a file. This output can be viewed later
    //     using GLVis: "glvis -m optimized -np num_mpi_tasks".
    {
       ostringstream mesh_name;
@@ -852,7 +828,7 @@ int main (int argc, char *argv[])
       pmesh->PrintAsOne(mesh_ofs);
    }
 
-   // 17. Compute the amount of energy decrease.
+   // 22. Compute the amount of energy decrease.
    const double fin_energy = a.GetParGridFunctionEnergy(x);
    double metric_part = fin_energy;
    if (lim_const > 0.0 || adapt_lim_const > 0.0)
@@ -875,7 +851,7 @@ int main (int argc, char *argv[])
            << (init_energy - fin_energy) * 100.0 / init_energy << " %." << endl;
    }
 
-   // 18. Visualize the final mesh and metric values.
+   // 23. Visualize the final mesh and metric values.
    if (visualization)
    {
       char title[] = "Final metric values";
@@ -889,7 +865,7 @@ int main (int argc, char *argv[])
                              600, 600, 300, 300);
    }
 
-   // 19. Visualize the mesh displacement.
+   // 23. Visualize the mesh displacement.
    if (visualization)
    {
       x0 -= x;
@@ -910,7 +886,7 @@ int main (int argc, char *argv[])
       }
    }
 
-   // 20. Free the used memory.
+   // 24. Free the used memory.
    delete S;
    delete target_c2;
    delete metric2;

tests/scripts/branch-history

@@ -20,10 +20,9 @@ sub usage {
   printf STDOUT <<EOF;
 
    $0 [-h|--help]
-   $0 [-b <branch>] {mfem_dir}
+   $0 {mfem_dir}
 
    where: {mfem_dir}  is the MFEM source directory [default value: ../..]
-          -b <branch> is the branch to check [default: HEAD]
           -h|--help   prints this usage information and exits
 
    This script checks if the current branch history, defined as the commits that
@@ -38,14 +37,16 @@ EOF
 }
 
 my $mfem_dir = "../..";
-my $branch = "HEAD";
+
+if (scalar @ARGV > 2) {
+  printf STDERR "Error: too many command line arguments\n";
+  usage 1;
+}
 
 while (my $opt = shift) {
   if ($opt) {
     if ($opt eq "-h" || $opt eq "--help") {
       usage 0;
-    } elsif ($opt eq "-b") {
-      $branch = shift;
     } else {
       $mfem_dir = $opt;
     }
@@ -72,7 +73,7 @@ my $max_branch_kb = 1000;
 my $status = 0; # Return code
 
 # Get SHA hash of all commits in this branch
-my @commits = split /\n/, `git log --pretty=format:%H master..$branch`;
+my @commits = split /\n/, `git log --pretty=format:%H master..HEAD`;
 
 # Check if total number of commits in this branch exceeds the maximum allowable
 my $ncommits = scalar @commits;
@@ -103,7 +104,7 @@ sub formatSize {
 # Loop over each commit in this branch, and check for large diffs
 my $total_size = 0;
 foreach my $sha (@commits) {
-  my @blobs = `git diff-tree -r -c --root --no-commit-id $sha`;
+  my @blobs = `git diff-tree -r --no-commit-id $sha`;
   my $nfiles_changed = scalar @blobs;
   if ($nfiles_changed > $commit_max_files_changed) {
     printf STDERR
@@ -116,53 +117,23 @@ foreach my $sha (@commits) {
   my $commit_size = 0;
   foreach my $blob (@blobs) {
     my @blob_split = (split /\s/, $blob);
-    my $nfields = scalar @blob_split;
-    my $fname = "(no-filename)";
+    my $src = @blob_split[2];
+    my $dst = @blob_split[3];
+    my $mode = @blob_split[4];
+    my $fname = @blob_split[5];
+
     my $blob_size = 0;
-
-    if ($nfields == 6) {
-      # 0 or 1 parents
-      my $src = @blob_split[2];
-      my $dst = @blob_split[3];
-      my $mode = @blob_split[4];
-      $fname = @blob_split[5];
-      # File was added
-      if ($mode eq "A") { $blob_size += int(`git cat-file -s $dst`); }
-      # File was copied
-      elsif ($mode =~ m/C\d*/) { $blob_size += int(`git cat-file -s $dst`); }
-      elsif ($mode eq "D") { }
-      # File was modified, use the gzip'ed diff as a proxy of the required git storage
-      elsif ($mode =~ m/M\d*/) { $blob_size += int(`git diff -U0 --binary $src $dst | gzip -c | wc -c`); }
-      elsif ($mode =~ m/R\d*/) { }
-      elsif ($mode eq "T") { }
-      elsif ($mode eq "U") { }
-      else { die "Unknown git status letter." }
-    } elsif ($nfields == 8) {
-      # 2 parents
-      my $src1 = @blob_split[3];
-      my $src2 = @blob_split[4];
-      my $dst = @blob_split[5];
-      my $mode = @blob_split[6];
-      $fname = @blob_split[7];
-      if ($mode eq "AA") {
-        # File was added
-        $blob_size += int(`git cat-file -s $dst`); }
-      elsif ($mode eq "DD") { }
-      elsif ($mode eq "MM") {
-        # File was modified, use the gzip'ed diff as a proxy of the required git
-        # storage
-        my $sz1 = int(`git diff -U0 --binary $src1 $dst | gzip -c | wc -c`);
-        my $sz2 = int(`git diff -U0 --binary $src2 $dst | gzip -c | wc -c`);
-        $blob_size += $sz1 < $sz2 ? $sz1 : $sz2; }
-      elsif ($mode eq "AM") {
-        my $sz2 = int(`git diff -U0 --binary $src2 $dst | gzip -c | wc -c`);
-        $blob_size += $sz2; }
-      elsif ($mode eq "MA") {
-        my $sz1 = int(`git diff -U0 --binary $src1 $dst | gzip -c | wc -c`);
-        $blob_size += $sz1; }
-      else { die "Unknown git status letter: $mode, commit: $sha, file: $fname.\n\t" }
-    }
-
+    # File was added
+    if ($mode eq "A") { $blob_size += int(`git cat-file -s $dst`); }
+    # File was copied
+    elsif ($mode =~ m/C\d*/) { $blob_size += int(`git cat-file -s $dst`); }
+    elsif ($mode eq "D") { }
+    # File was modified, use the gzip'ed diff as a proxy of the required git storage
+    elsif ($mode =~ m/M\d*/) { $blob_size += int(`git diff -U0 --binary $src $dst | gzip -c | wc -c`); }
+    elsif ($mode =~ m/R\d*/) { }
+    elsif ($mode eq "T") { }
+    elsif ($mode eq "U") { }
+    else { die "Unknown git status letter." }
     if ($blob_size > $max_blob_kb*1024) {
       $status = 1;
       printf STDERR "\033[31mLarge change of size %s in file %s.\033[0m\n", formatSize($blob_size), $fname;
@@ -184,7 +155,6 @@ if ($total_size > $max_branch_kb*1024) {
 }
 
 if ($status) {
-  printf STDERR "\033[36mBranch $branch has errors.\033[0m\n";
   chdir $cur_dir;
   my $testname = basename $0;
   open(my $f, '>', "$testname.msg");

tests/scripts/documentation

@@ -43,8 +43,8 @@ cd $mfem_dir
 # Test the documentation of some make targets
 make help
 make distclean
-# make config MFEM_USE_MPI=YES
-# make status
+make config MFEM_USE_MPI=YES
+make status
 
 # Test the build of the Doxygen documentation
 cd doc; make clean; make

tests/unit/CMakeLists.txt

@@ -25,9 +25,7 @@ set(UNIT_TESTS_SRCS
   linalg/test_ilu.cpp
   linalg/test_matrix_block.cpp
   linalg/test_matrix_dense.cpp
-  linalg/test_matrix_hypre.cpp
   linalg/test_matrix_rectangular.cpp
-  linalg/test_matrix_sparse.cpp
   linalg/test_matrix_square.cpp
   linalg/test_ode.cpp
   linalg/test_ode2.cpp
@@ -35,7 +33,6 @@ set(UNIT_TESTS_SRCS
   linalg/test_cg_indefinite.cpp
   linalg/test_vector.cpp
   mesh/test_mesh.cpp
-  mesh/test_ncmesh.cpp
   fem/test_1d_bilininteg.cpp
   fem/test_2d_bilininteg.cpp
   fem/test_3d_bilininteg.cpp

tests/unit/fem/test_assembly_levels.cpp

@@ -48,6 +48,7 @@ void test_assembly_level(Mesh &&mesh, int order, bool dg, const int pb,
                          const AssemblyLevel assembly)
 {
    mesh.EnsureNodes();
+   mesh.SetCurvature(mesh.GetNodalFESpace()->GetOrder(0));
    int dim = mesh.Dimension();
 
    FiniteElementCollection *fec;
@@ -106,77 +107,13 @@ void test_assembly_level(Mesh &&mesh, int order, bool dg, const int pb,
 
 TEST_CASE("Assembly Levels", "[AssemblyLevel]")
 {
-   SECTION("Continuous Galerkin")
+   for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
    {
-      const bool dg = false;
-      SECTION("2D")
+      for (int pb : {0, 1, 2})
       {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
+         for (bool dg : {true, false})
          {
-            for (int pb : {0, 1, 2})
-            {
-               for (int order : {2, 3, 4})
-               {
-                  test_assembly_level(Mesh("../../data/inline-quad.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/periodic-hexagon.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/star-q3.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-               }
-            }
-         }
-      }
-      SECTION("3D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               int order = 2;
-               test_assembly_level(Mesh("../../data/inline-hex.mesh", 1, 1),
-                                   order, dg, pb, assembly);
-               test_assembly_level(Mesh("../../data/fichera-q3.mesh", 1, 1),
-                                   order, dg, pb, assembly);
-            }
-         }
-      }
-      SECTION("AMR 2D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               for (int order : {2, 3, 4})
-               {
-                  test_assembly_level(Mesh("../../data/amr-quad.mesh", 1, 1),
-                                      order, false, 0, assembly);
-               }
-            }
-         }
-      }
-      SECTION("AMR 3D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
-            {
-               int order = 2;
-               test_assembly_level(Mesh("../../data/fichera-amr.mesh", 1, 1),
-                                   order, false, 0, assembly);
-            }
-         }
-      }
-   }
-
-   SECTION("Discontinuous Galerkin")
-   {
-      const bool dg = true;
-      SECTION("2D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
+            SECTION("2D")
             {
                for (int order : {2, 3, 4})
                {
@@ -188,24 +125,32 @@ TEST_CASE("Assembly Levels", "[AssemblyLevel]")
                                       order, dg, pb, assembly);
                }
             }
-         }
-      }
-      SECTION("3D")
-      {
-         for (AssemblyLevel assembly : {AssemblyLevel::PARTIAL,AssemblyLevel::ELEMENT,AssemblyLevel::FULL})
-         {
-            for (int pb : {0, 1, 2})
+
+            SECTION("3D")
             {
-               for (bool dg : {true, false})
-               {
-                  int order = 2;
-                  test_assembly_level(Mesh("../../data/periodic-cube.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-                  test_assembly_level(Mesh("../../data/fichera-q3.mesh", 1, 1),
-                                      order, dg, pb, assembly);
-               }
+               int order = 2;
+               test_assembly_level(Mesh("../../data/periodic-cube.mesh", 1, 1),
+                                   order, dg, pb, assembly);
+               test_assembly_level(Mesh("../../data/fichera-q3.mesh", 1, 1),
+                                   order, dg, pb, assembly);
             }
          }
+
+         // Test AMR cases (DG not implemented)
+         SECTION("AMR 2D")
+         {
+            for (int order : {2, 3, 4})
+            {
+               test_assembly_level(Mesh("../../data/amr-quad.mesh", 1, 1),
+                                   order, false, 0, assembly);
+            }
+         }
+         SECTION("AMR 3D")
+         {
+            int order = 2;
+            test_assembly_level(Mesh("../../data/fichera-amr.mesh", 1, 1),
+                                order, false, 0, assembly);
+         }
       }
    }
 } // test case

tests/unit/fem/test_pa_coeff.cpp

@@ -479,8 +479,25 @@ TEST_CASE("Hcurl/Hdiv pa_coeff")
                      {
                         if (spaceType == Hcurl)
                         {
-                           paform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff2));
-                           assemblyform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff2));
+                           const FiniteElement *fel = fespace.GetFE(0);
+                           const IntegrationRule *intRule = &MassIntegrator::GetRule(*fel, *fel,
+                                                                                     *mesh->GetElementTransformation(0));
+
+                           if (coeffType >= 3 && dimension == 3)
+                           {
+                              paform.AddDomainIntegrator(new CurlCurlIntegrator(*smcoeff, intRule));
+                              assemblyform.AddDomainIntegrator(new CurlCurlIntegrator(*mcoeff, intRule));
+                           }
+                           else if (coeffType == 2 && dimension == 3)
+                           {
+                              paform.AddDomainIntegrator(new CurlCurlIntegrator(*vcoeff, intRule));
+                              assemblyform.AddDomainIntegrator(new CurlCurlIntegrator(*vcoeff, intRule));
+                           }
+                           else
+                           {
+                              paform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff2));
+                              assemblyform.AddDomainIntegrator(new CurlCurlIntegrator(*coeff2));
+                           }
                         }
                         else
                         {

tests/unit/fem/test_pa_kernels.cpp

@@ -422,9 +422,9 @@ void test_pa_convection(Mesh &&mesh, int order, bool dg)
 //Basic unit test for convection
 TEST_CASE("PA Convection", "[PartialAssembly]")
 {
-   SECTION("2D")
+   for (bool dg : {true, false})
    {
-      for (bool dg : {true, false})
+      SECTION("2D")
       {
          for (int order : {2, 3, 4})
          {
@@ -433,10 +433,8 @@ TEST_CASE("PA Convection", "[PartialAssembly]")
             test_pa_convection(Mesh("../../data/star-q3.mesh", 1, 1), order, dg);
          }
       }
-   }
-   SECTION("3D")
-   {
-      for (bool dg : {true, false})
+
+      SECTION("3D")
       {
          int order = 2;
          test_pa_convection(Mesh("../../data/periodic-cube.mesh", 1, 1), order, dg);
@@ -444,9 +442,9 @@ TEST_CASE("PA Convection", "[PartialAssembly]")
       }
    }
    // Test AMR cases (DG not implemented)
-   SECTION("AMR 2D")
+   for (int order : {2, 3, 4})
    {
-      for (int order : {2, 3, 4})
+      SECTION("AMR 2D")
       {
          test_pa_convection(Mesh("../../data/amr-quad.mesh", 1, 1), order, false);
       }

tests/unit/linalg/test_matrix_dense.cpp

@@ -289,7 +289,7 @@ TEST_CASE("DenseTensor LinearSolve methods",
    {
       for (int r=0; r<N; ++r)
       {
-         REQUIRE(xans_batch(r,e) == Approx(X[r]));
+         REQUIRE(xans_batch(r,e) == X[r]);
       }
    }
 }

tests/unit/linalg/test_matrix_hypre.cpp

@@ -1,106 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "catch.hpp"
-#include "mfem.hpp"
-
-namespace mfem
-{
-
-constexpr double EPS = 1.e-12;
-
-#ifdef MFEM_USE_MPI
-
-TEST_CASE("HypreParMatrixAbsMult",  "[Parallel], [HypreParMatrixAbsMult]")
-{
-   int rank;
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-   int dim = 2;
-   int ne = 4;
-   for (int order = 1; order <= 3; ++order)
-   {
-      Mesh * mesh = new Mesh(ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
-      ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-      delete mesh;
-      FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
-      FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
-      ParFiniteElementSpace R_space(pmesh, hdiv_coll);
-      ParFiniteElementSpace W_space(pmesh, l2_coll);
-
-      int n = R_space.GetTrueVSize();
-      int m = W_space.GetTrueVSize();
-      ParMixedBilinearForm a(&R_space, &W_space);
-      a.AddDomainIntegrator(new VectorFEDivergenceIntegrator);
-      a.Assemble();
-      a.Finalize();
-
-      HypreParMatrix *A = a.ParallelAssemble();
-      HypreParMatrix *Aabs = new HypreParMatrix(*A);
-
-      hypre_ParCSRMatrix * AparCSR = *Aabs;
-
-      int nnzd = AparCSR->diag->num_nonzeros;
-      for (int j = 0; j < nnzd; j++)
-      {
-         AparCSR->diag->data[j] = fabs(AparCSR->diag->data[j]);
-      }
-
-      int nnzoffd = AparCSR->offd->num_nonzeros;
-      for (int j = 0; j < nnzoffd; j++)
-      {
-         AparCSR->offd->data[j] = fabs(AparCSR->offd->data[j]);
-      }
-
-      Vector X0(n), X1(n);
-      Vector Y0(m), Y1(m);
-
-      X0.Randomize();
-      Y0.Randomize(1);
-      Y1.Randomize(1);
-      A->AbsMult(3.4,X0,-2.3,Y0);
-      Aabs->Mult(3.4,X0,-2.3,Y1);
-
-      Y1 -=Y0;
-      double error = Y1.Norml2();
-
-      std::cout << "Testing AbsMult:   order: " << order
-                << ", error norm on rank "
-                << rank << ": " << error << std::endl;
-
-      REQUIRE(error == Approx(EPS));
-
-      MPI_Barrier(MPI_COMM_WORLD);
-
-      Y0.Randomize();
-      X0.Randomize(1);
-      X1.Randomize(1);
-      A->AbsMultTranspose(3.4,Y0,-2.3,X0);
-      Aabs->MultTranspose(3.4,Y0,-2.3,X1);
-      X1 -=X0;
-
-      error = X1.Norml1();
-      std::cout << "Testing AbsMultT:  order: " << order
-                << ", error norm on rank "
-                << rank << ": " << error << std::endl;
-
-      REQUIRE(error == Approx(EPS));
-
-      delete A;
-      delete Aabs;
-      delete hdiv_coll;
-      delete l2_coll;
-      delete pmesh;
-   }
-}
-
-#endif // MFEM_USE_MPI
-
-} // namespace mfem

tests/unit/linalg/test_matrix_sparse.cpp

@@ -1,89 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "catch.hpp"
-#include "mfem.hpp"
-
-namespace mfem
-{
-
-constexpr double EPS = 1.e-12;
-
-TEST_CASE("SparseMatrixAbsMult", "[SparseMatrixAbsMult]")
-{
-   int dim = 2;
-   int ne = 4;
-   for (int order = 1; order <= 3; ++order)
-   {
-      Mesh * mesh = new Mesh(ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
-      FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
-      FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
-      FiniteElementSpace R_space(mesh, hdiv_coll);
-      FiniteElementSpace W_space(mesh, l2_coll);
-
-      int n = R_space.GetTrueVSize();
-      int m = W_space.GetTrueVSize();
-      MixedBilinearForm a(&R_space, &W_space);
-      a.AddDomainIntegrator(new VectorFEDivergenceIntegrator);
-      a.Assemble();
-      a.Finalize();
-
-      SparseMatrix &A = a.SpMat();
-      SparseMatrix *Aabs = new SparseMatrix(A);
-
-      int nnz = Aabs->NumNonZeroElems();
-      for (int j = 0; j < nnz; j++)
-      {
-         Aabs->GetData()[j] = fabs(Aabs->GetData()[j]);
-      }
-
-      Vector X0(n), X1(n);
-      Vector Y0(m), Y1(m);
-
-      X0.Randomize();
-      Y0.Randomize(1);
-      Y1.Randomize(1);
-      A.AbsMult(X0,Y0);
-      Aabs->Mult(X0,Y1);
-
-      Y1 -=Y0;
-      double error = Y1.Norml2();
-
-      std::cout << "Testing AbsMult:   order: " << order
-                << ", error norm: "
-                << error << std::endl;
-
-      REQUIRE(error == Approx(EPS));
-
-      Y0.Randomize();
-      X0.Randomize(1);
-      X1.Randomize(1);
-      A.AbsMultTranspose(Y0,X0);
-      Aabs->MultTranspose(Y0,X1);
-      X1 -=X0;
-
-      error = X1.Norml2();
-
-      std::cout << "Testing AbsMultT:  order: " << order
-                << ", error norm: "
-                << error << std::endl;
-
-      REQUIRE(error == Approx(EPS));
-
-      delete Aabs;
-      delete hdiv_coll;
-      delete l2_coll;
-      delete mesh;
-   }
-}
-
-
-} // namespace mfem

tests/unit/mesh/test_ncmesh.cpp

@@ -1,241 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "mfem.hpp"
-#include "catch.hpp"
-
-namespace mfem
-{
-
-constexpr double EPS = 1.e-12;
-
-// Test case: Verify that a conforming mesh yields the same norm for the
-//            assembled diagonal with PA when using the standard (conforming)
-//            Mesh vs. the corresponding (non-conforming) NCMesh.
-//            (note: permutations of the values in the diagonal are expected)
-TEST_CASE("NCMesh PA diagonal", "[NCMesh]")
-{
-   SECTION("Quad mesh")
-   {
-      int ne = 2;
-      Mesh mesh(ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
-      Mesh nc_mesh(ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
-      nc_mesh.EnsureNCMesh();
-
-      mesh.UniformRefinement();
-      nc_mesh.UniformRefinement();
-
-      int dim = 2;
-      for (int order = 1; order <= 3; ++order)
-      {
-         ND_FECollection fec(order, dim);
-
-         FiniteElementSpace fes(&mesh, &fec);
-         FiniteElementSpace nc_fes(&nc_mesh, &fec);
-
-         BilinearForm a(&fes);
-         BilinearForm nc_a(&nc_fes);
-
-         a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-         nc_a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-
-         ConstantCoefficient coef(1.0);
-         a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-         nc_a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-
-         a.Assemble();
-         nc_a.Assemble();
-
-         Vector diag(fes.GetTrueVSize());
-         Vector nc_diag(nc_fes.GetTrueVSize());
-         a.AssembleDiagonal(diag);
-         nc_a.AssembleDiagonal(nc_diag);
-
-         double error = fabs(diag.Norml2() - nc_diag.Norml2());
-         std::cout << "Testing quad NCMesh PA diag:    "
-                   "order: " << order << ", error: " << error << std::endl;
-         REQUIRE(error == Approx(EPS));
-      }
-   }
-
-   SECTION("Hexa mesh")
-   {
-      int ne = 2;
-      Mesh mesh(ne, ne, ne, Element::HEXAHEDRON, 1, 1.0, 1.0, 1.0);
-      Mesh nc_mesh(ne, ne, ne, Element::HEXAHEDRON, 1, 1.0, 1.0, 1.0);
-      nc_mesh.EnsureNCMesh();
-
-      mesh.UniformRefinement();
-      nc_mesh.UniformRefinement();
-
-      int dim = 3;
-      for (int order = 1; order <= 3; ++order)
-      {
-         ND_FECollection fec(order, dim);
-
-         FiniteElementSpace fes(&mesh, &fec);
-         FiniteElementSpace nc_fes(&nc_mesh, &fec);
-
-         BilinearForm a(&fes);
-         BilinearForm nc_a(&nc_fes);
-
-         a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-         nc_a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-
-         ConstantCoefficient coef(1.0);
-         a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-         nc_a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-
-         a.Assemble();
-         nc_a.Assemble();
-
-         Vector diag(fes.GetTrueVSize());
-         Vector nc_diag(nc_fes.GetTrueVSize());
-         a.AssembleDiagonal(diag);
-         nc_a.AssembleDiagonal(nc_diag);
-
-         double error = fabs(diag.Sum() - nc_diag.Sum());
-         std::cout << "Testing hexa NCMesh PA diag:    "
-                   "order: " << order << ", error: " << error << std::endl;
-         REQUIRE(error == Approx(EPS));
-      }
-   }
-
-} // test case
-
-#ifdef MFEM_USE_MPI
-
-// Test case: Verify that a conforming mesh yields the same norm for the
-//            assembled diagonal with PA when using the standard (conforming)
-//            Mesh vs. the corresponding (non-conforming) NCMesh.
-//            (note: permutations of the values in the diagonal are expected)
-TEST_CASE("pNCMesh PA diagonal",  "[Parallel], [NCMesh]")
-{
-   int rank;
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-   SECTION("Quad pmesh")
-   {
-      int ne = 2;
-      Mesh mesh(ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
-      Mesh nc_mesh(ne, ne, Element::QUADRILATERAL, 1, 1.0, 1.0);
-      nc_mesh.EnsureNCMesh();
-
-      mesh.UniformRefinement();
-      nc_mesh.UniformRefinement();
-
-      ParMesh pmesh(MPI_COMM_WORLD, mesh);
-      ParMesh nc_pmesh(MPI_COMM_WORLD, nc_mesh);
-
-      int dim = 2;
-      for (int order = 1; order <= 3; ++order)
-      {
-         ND_FECollection fec(order, dim);
-
-         ParFiniteElementSpace pfes(&pmesh, &fec);
-         ParFiniteElementSpace nc_pfes(&nc_pmesh, &fec);
-
-         ParBilinearForm a(&pfes);
-         ParBilinearForm nc_a(&nc_pfes);
-
-         a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-         nc_a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-
-         ConstantCoefficient coef(1.0);
-         a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-         nc_a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-
-         a.Assemble();
-         nc_a.Assemble();
-
-         Vector diag(pfes.GetTrueVSize());
-         Vector nc_diag(nc_pfes.GetTrueVSize());
-         a.AssembleDiagonal(diag);
-         nc_a.AssembleDiagonal(nc_diag);
-
-         double diag_lsum = diag.Sum(), nc_diag_lsum = nc_diag.Sum();
-         double diag_gsum = 0.0, nc_diag_gsum = 0.0;
-         MPI_Allreduce(&diag_lsum, &diag_gsum, 1, MPI_DOUBLE, MPI_SUM,
-                       MPI_COMM_WORLD);
-         MPI_Allreduce(&nc_diag_lsum, &nc_diag_gsum, 1, MPI_DOUBLE, MPI_SUM,
-                       MPI_COMM_WORLD);
-         double error = fabs(diag_gsum - nc_diag_gsum);
-         if (rank==0)
-         {
-            std::cout << "Testing quad pNCMesh PA diag:    "
-                      "order: " << order << ", error: " << error << std::endl;
-         }
-         REQUIRE(error == Approx(EPS));
-         MPI_Barrier(MPI_COMM_WORLD);
-      }
-   }
-
-   SECTION("Hexa pmesh")
-   {
-      int ne = 2;
-      Mesh mesh(ne, ne, ne, Element::HEXAHEDRON, 1, 1.0, 1.0, 1.0);
-      Mesh nc_mesh(ne, ne, ne, Element::HEXAHEDRON, 1, 1.0, 1.0, 1.0);
-      nc_mesh.EnsureNCMesh();
-
-      mesh.UniformRefinement();
-      nc_mesh.UniformRefinement();
-
-      ParMesh pmesh(MPI_COMM_WORLD, mesh);
-      ParMesh nc_pmesh(MPI_COMM_WORLD, nc_mesh);
-
-      int dim = 3;
-      for (int order = 1; order <= 3; ++order)
-      {
-         ND_FECollection fec(order, dim);
-
-         ParFiniteElementSpace pfes(&pmesh, &fec);
-         ParFiniteElementSpace nc_pfes(&nc_pmesh, &fec);
-
-         ParBilinearForm a(&pfes);
-         ParBilinearForm nc_a(&nc_pfes);
-
-         a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-         nc_a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-
-         ConstantCoefficient coef(1.0);
-         a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-         nc_a.AddDomainIntegrator(new CurlCurlIntegrator(coef));
-
-         a.Assemble();
-         nc_a.Assemble();
-
-         Vector diag(pfes.GetTrueVSize());
-         Vector nc_diag(nc_pfes.GetTrueVSize());
-         a.AssembleDiagonal(diag);
-         nc_a.AssembleDiagonal(nc_diag);
-
-         double diag_lsum = diag.Sum(), nc_diag_lsum = nc_diag.Sum();
-         double diag_gsum = 0.0, nc_diag_gsum = 0.0;
-         MPI_Allreduce(&diag_lsum, &diag_gsum, 1, MPI_DOUBLE, MPI_SUM,
-                       MPI_COMM_WORLD);
-         MPI_Allreduce(&nc_diag_lsum, &nc_diag_gsum, 1, MPI_DOUBLE, MPI_SUM,
-                       MPI_COMM_WORLD);
-         double error = fabs(diag_gsum - nc_diag_gsum);
-         if (rank==0)
-         {
-            std::cout << "Testing hexa pNCMesh PA diag:    "
-                      "order: " << order << ", error: " << error << std::endl;
-         }
-         REQUIRE(error == Approx(EPS));
-         MPI_Barrier(MPI_COMM_WORLD);
-      }
-   }
-
-} // test case
-
-#endif // MFEM_USE_MPI
-
-} // namespace mfem