complex-jacobi minor cleanup

BlockOperator on device

.appveyor.yml

@@ -16,7 +16,7 @@ install:
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%
 
 # Install METIS
-- ps: Start-FileDownload 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz'
+- ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
@@ -26,17 +26,17 @@ install:
 - cd ..
 
 # Install hypre
-- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
-- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
-- cd hypre-2-10-0b
-- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
+- 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2.19.0/src
+- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
-- cd ..
+- cd ../..
 
 # MFEM
 before_build:
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
 
 build_script:

.gitignore

@@ -29,6 +29,8 @@ config/sample-runs-build.log
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
+doc/undoc.log
+doc/warnings.log
 
 # Temporary files created by the tests.
 *.stderr
@@ -48,6 +50,7 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
+examples/ex25-gpu
 
 examples/refined.mesh
 examples/displaced.mesh
@@ -120,7 +123,7 @@ examples/sundials/ex16-final.*
 examples/sundials/Example16*
 
 examples/petsc/ex[1-69]p
-examples/petsc/ex10p
+examples/petsc/ex1[0-1]p
 
 examples/petsc/mesh.*
 examples/petsc/sol.*
@@ -135,6 +138,7 @@ examples/petsc/Example9*
 examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
+examples/petsc/mode_*
 
 examples/pumi/ex1
 examples/pumi/ex[126]p
@@ -167,6 +171,7 @@ miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
+miniapps/meshing/trimmer
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
@@ -180,6 +185,7 @@ miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
+miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 
@@ -239,6 +245,9 @@ miniapps/navier/navier_3dfoc
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output
 
+miniapps/adjoint/cvsRoberts_ASAi_dns
+miniapps/adjoint/adjoint_advection_diffusion
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

.travis.yml

@@ -16,6 +16,12 @@ stages:
   - tests
   - optional
 
+env:
+  global:
+    - HYPRE_ARCHIVE=v2.19.0.tar.gz
+      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
+      HYPRE_TOP_DIR=hypre-2.19.0
+
 jobs:
   include:
 
@@ -138,8 +144,7 @@ jobs:
            NPROCS=2
       cache:
         directories:
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
       before_cache:
         - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -169,8 +174,7 @@ jobs:
            NPROCS=2
       cache:
         directories:
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
       before_cache:
         - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -193,7 +197,7 @@ jobs:
         - cd ${TRAVIS_BUILD_DIR}/build
         - cmake ..
              -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../hypre-2.10.0b/src/hypre
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
              -DMFEM_MPI_NP=$NPROCS
         - make -j3 mfem examples
         - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
@@ -201,8 +205,7 @@ jobs:
         - ctest --output-on-failure
       cache:
         directories:
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
       before_cache:
         - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -247,8 +250,7 @@ jobs:
            TMPDIR=/tmp
       cache:
         directories:
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
           - $HOME/local-cached
       before_cache:
@@ -268,8 +270,7 @@ jobs:
            TMPDIR=/tmp
       cache:
         directories:
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
-          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
           - $TRAVIS_BUILD_DIR/../metis-4.0
           - $HOME/local-cached
       before_cache:
@@ -335,18 +336,18 @@ install:
 
    # hypre
    - if [ $MPI == "YES" ]; then
-        if [ ! -e hypre-2.10.0b/src/hypre/lib/libHYPRE.a ]; then
-           wget https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz --no-check-certificate;
-           rm -rf hypre-2.10.0b;
-           tar xvzf hypre-2.10.0b.tar.gz;
-           cd hypre-2.10.0b/src;
-           ./configure --disable-fortran --without-fei CC=mpicc CXX=mpic++;
+        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
+           wget $HYPRE_URL;
+           rm -rf $HYPRE_TOP_DIR;
+           tar xvzf $HYPRE_ARCHIVE;
+           cd $HYPRE_TOP_DIR/src;
+           ./configure --disable-fortran CC=mpicc CXX=mpic++;
            make -j3;
            cd ../..;
         else
-            echo "Reusing cached hypre-2.10.0b/";
+            echo "Reusing cached $HYPRE_TOP_DIR/";
         fi;
-        ln -s hypre-2.10.0b hypre;
+        ln -s $HYPRE_TOP_DIR hypre;
      else
         echo "Serial build, not using hypre";
      fi
@@ -354,7 +355,7 @@ install:
    # METIS
    - if [ $MPI == "YES" ]; then
         if [ ! -e metis-4.0/libmetis.a ]; then
-           wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz;
+           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
            tar xvzf metis-4.0.3.tar.gz;
            make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
            rm -rf metis-4.0;

CHANGELOG

@@ -24,8 +24,17 @@ Meshing improvements
   and orientation based metrics.
 
 - Added support for r-adaptivity with more than one discrete field. This allows
-  the user to specify different discrete functions for controlling the 
-  size, aspect-ratio, orientation, and skew of elements in the mesh. 
+  the user to specify different discrete functions for controlling the
+  size, aspect-ratio, orientation, and skew of elements in the mesh.
+
+- Added TMOP capability for approximate tangential mesh relaxation.
+
+- Added support for reading periodic meshes in Gmsh format (version 2.2). See
+  for example the periodic-annulus-sector and periodic-torus-sector files in
+  the data directory.
+
+- Added complete action of the TMOP Integrator to account for the spatial
+  derivatives of discrete and analytic targets.
 
 Performance improvements
 ------------------------
@@ -38,8 +47,23 @@ Performance improvements
   These are now enabled by default, and can be disabled with MFEM_USE_SIMD=NO.
   See the new file linalg/simd.hpp and the new directory linalg/simd.
 
+Improved GPU capabilities
+-------------------------
 - Added support for Chebyshev accelerated polynomial smoother on GPU.
 
+- Optimized AMD/HIP kernel support.
+
+- Added a Full Assembly mode compatible with Device kernel execution. This
+  assembly level builds on top of the current Element Assembly kernels to
+  compute a global sparse matrix. All integrators supported by element assembly
+  are also supported by full assembly. See the '-fa' option in Example 9.
+
+- Added support for BlockOperator on GPU. See the updated Example 5.
+
+- Added partial assembly and GPU support for ComplexOperator,
+  [Par]ComplexGridFunction, [Par]ComplexLinearForm, and [Par]SesquilinearForm.
+  See the updated Example 22.
+
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -63,9 +87,11 @@ Discretization improvements
   and, in the continuous field case, arbitrary mesh edges and faces.
 
 - Added new coefficient and vector coefficient classes for QuadratureFunctions.
-  Additionaly, new LinearForm integrators were also added which make use of
+  Additionally, new LinearForm integrators were also added which make use of
   these new QuadratureFunction coefficient classes.
 
+- Added support face integrals on the boundaries of NURBS meshes.
+
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -78,6 +104,10 @@ Linear and nonlinear solvers
   and solution during the solving process of an IterativeSolver after every
   iteration.
 
+- Added support for the CVODES package in SUNDIALS which provides ODE
+  solvers with sensitivity analysis capabilities. See the CVODESSolver
+  class and the new adjoint miniapps below.
+
 - Block arrays of parallel matrices can now be merged into a single parallel
   matrix with the function HypreParMatrixFromBlocks. This could be useful for
   solving block systems with parallel direct solvers such as STRUMPACK.
@@ -85,6 +115,8 @@ Linear and nonlinear solvers
 - In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
   solver can work with non-SPD preconditioner B.
 
+- Added support for the SLEPc eigensolver package.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -94,11 +126,25 @@ New and updated examples and miniapps
 - Added a new Example 26/26p to demonstrate the construction of a matrix-free
   geometric and p-multigrid preconditioner for the Laplace problem.
 
-- Added a new example, Example 27/27p, to demonstrate the enforcement of
-  various boundary conditions with the Laplace operator.  The example shows the
-  procedures for applying Dirichlet, Neumann (both homogeneous and
-  inhomogeneous), Robin, and periodic boundary conditions with either H1 or DG
-  discretizations.
+- Added a new example, Example 27/27p, to demonstrate the enforcement of various
+  boundary conditions with the Laplace operator. The example shows the procedure
+  for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
+  and periodic boundary conditions with either H1 or DG discretizations.
+
+- Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
+  equations of incompressible fluid dynamics. See the miniapps/navier directory
+  for more details.
+
+- Added a new miniapps/adjoint directory with two miniapps demonstrating how to
+  solve adjoint problems in MFEM using the CVODES package in SUNDIALS. Both of
+  these miniapps require the MFEM_USE_SUNDIALS configuration option.
+  * The cvsRoberts_ASAi_dns miniapp solves a backward adjoint problem for a
+    system of ODEs, evaluating both forward and adjoint quadratures in serial.
+  * The adjoint_advection_diffusion miniapp solves a backward adjoint problem
+    for an advection diffusion PDE, evaluating adjoint quadratures in parallel.
+
+- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
+  equation with the shift-and-invert spectral transformation method.
 
 - Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
   stitching together opposite surfaces of a mesh to create a topologically
@@ -107,11 +153,19 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
   the Dirichlet problem for the minimal surface equation.
 
-- Added partial assembly support to examples 4/4p and 5/5p, with diagonal
-  preconditioning.
+- Added full assembly support in Example 9/9p.
 
-- Added a new test problem in example 24/24p, demonstrating a mixed bilinear
-  form for H(div) and L_2, with partial assembly support.
+- Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
+  form for H1, H(curl), H(div) and L_2, with partial assembly support.
+
+- Added weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
+
+- Added a simple mesh editing miniapp, Trimmer, which trims away portions of a
+  mesh based on element attributes. Any newly exposed boundary elements are
+  assigned attribute numbers related to the trimmed element attributes.
+
+- Added partial assembly and device support to Example 4/4p, Example 5/5p,
+  Example 22/22p, and Example 25/25p, with diagonal preconditioning.
 
 Improved testing
 ----------------

CMakeLists.txt

@@ -149,9 +149,13 @@ if (MFEM_USE_MPI)
       message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
     endif()
     set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
+    if (MFEM_USE_SLEPC)
+      find_package(SLEPc REQUIRED config)
+      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
+    endif()
   endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
   foreach(PKG IN LISTS PKGS_NEED_MPI)
     if (MFEM_USE_${PKG})
       message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -207,10 +211,10 @@ endif()
 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
   if (NOT MFEM_USE_MPI)
-    find_package(SUNDIALS REQUIRED NVector_Serial CVODE ARKODE KINSOL)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
   else()
     find_package(SUNDIALS REQUIRED
-      NVector_Serial NVector_Parallel NVector_ParHyp CVODE ARKODE KINSOL)
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
   endif()
 endif()
 
@@ -352,7 +356,7 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")

CONTRIBUTING.md

@@ -109,6 +109,7 @@ The MFEM source code has the following structure:
   ├── linalg
   ├── mesh
   ├── miniapps
+  │   ├── adjoint
   │   ├── common
   │   ├── electromagnetics
   │   ├── gslib

INSTALL

@@ -383,6 +383,10 @@ MFEM_USE_PETSC = YES/NO
    and other features based on the PETSc package. When enabled, this option uses
    the PETSC_* library options, see below.
 
+MFEM_USE_SLEPC = YES/NO
+   Enable MFEM eigensolvers based on the SLEPc package. When enabled, this
+   option uses the SLEPC_* library options, see below.
+
 MFEM_USE_MPFR = YES/NO
    MPFR is a library for multiple-precision floating-point computations. This
    option enables the use of MPFR in MFEM, e.g. for precise computation of 1D
@@ -597,6 +601,12 @@ The specific libraries and their options are:
   Options: PETSC_OPT, PETSC_LIB.
   Versions: PETSc >= 3.8.0.
 
+- SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
+  uses some of the PETSc options when compiled.
+  URL: https://slepc.upv.es/
+  Options: SLEPC_OPT, SLEPC_LIB.
+  Versions: SLEPc >= 3.8.0.
+
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
   Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
   URL: https://github.com/LLNL/axom
@@ -649,12 +659,11 @@ The specific libraries and their options are:
   Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
   Versions: OCCA >= 1.0.9.
 
-- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
-  or later version, specifically, git-hash 3d05795 or later.
+- libCEED (optional), used when MFEM_USE_CEED = YES.
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.6.
+  Versions: libCEED >= 0.6, git-hash a970f63.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.

config/XSDKDefaults.cmake

@@ -244,6 +244,10 @@ IF (DEFINED TPL_ENABLE_PETSC)
   SET(MFEM_USE_PETSC ${TPL_ENABLE_PETSC} CACHE BOOL "Enable PETSc support." FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_SLEPC)
+  SET(MFEM_USE_SLEPC ${TPL_ENABLE_SLEPC} CACHE BOOL "Enable SLEPc support." FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_MPFR)
   SET(MFEM_USE_MPFR ${TPL_ENABLE_MPFR} CACHE BOOL "Enable MPFR usage." FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -38,6 +38,7 @@ set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
+set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)

config/cmake/config.hpp.in

@@ -104,6 +104,9 @@
 // Enable MFEM functionality based on the PETSc library
 #cmakedefine MFEM_USE_PETSC
 
+// Enable MFEM functionality based on the SLEPc library
+#cmakedefine MFEM_USE_SLEPC
+
 // Enable MFEM functionality based on the Sidre library
 #cmakedefine MFEM_USE_SIDRE
 

config/cmake/modules/FindSLEPc.cmake

@@ -0,0 +1,44 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Sets the following variables:
+#   - SLEPC_FOUND
+#   - SLEPC_INCLUDE_DIRS
+#   - SLEPC_LIBRARIES
+
+set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
+  "Additional packages required by SLEPc")
+
+include(MfemCmakeUtilities)
+mfem_find_package(SLEPc SLEPC SLEPC_DIR
+  "include" "slepceps.h"
+  "${PETSC_ARCH}/lib" "slepc" # add NAMES_PER_DIR?
+  "Paths to headers required by SLEPc."
+  "Libraries required by SLEPc."
+  ADD_COMPONENT "config" "${PETSC_ARCH}/include" "slepcconf.h" "" ""
+  CHECK_BUILD SLEPC_VERSION_OK TRUE
+"
+#include \"petsc.h\"
+#include \"slepceps.h\"
+int main()
+{
+  PetscErrorCode ierr;
+  int argc = 0;
+  char** argv = NULL;
+  ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
+  EPS eps;
+  ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr);
+  ierr = EPSDestroy(&eps); CHKERRQ(ierr);
+  ierr = SlepcFinalize(); CHKERRQ(ierr);
+  return 0;
+}
+"
+  )

config/cmake/modules/FindSUNDIALS.cmake

@@ -25,5 +25,6 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
   ADD_COMPONENT NVector_ParHyp
     "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
   ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
+  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
   ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
   ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -731,7 +731,7 @@ function(mfem_export_mk_files)
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
       MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
       MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
   foreach(var ${CONFIG_MK_BOOL_VARS})

config/config.hpp

@@ -48,6 +48,9 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_SLEPC
+#error Building with SLEPc (MFEM_USE_SLEPC=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_PUMI
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif

config/config.hpp.in

@@ -118,6 +118,9 @@
 // Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC
 
+// Enable functionality based on the SLEPc library
+// #define MFEM_USE_SLEPC
+
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 

config/config.mk.in

@@ -37,6 +37,7 @@ MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
+MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@

config/defaults.cmake

@@ -39,6 +39,7 @@ option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
+option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
@@ -87,6 +88,8 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."
 
 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")
 
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
+# and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
     "Path to the SUNDIALS library.")
 # The following may be necessary, if SUNDIALS was built with KLU:
@@ -155,6 +158,10 @@ set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
     "Path to the PETSc main directory.")
 set(PETSC_ARCH "arch-linux2-c-debug" CACHE STRING "PETSc build architecture.")
 
+set(SLEPC_DIR "${MFEM_DIR}/../slepc" CACHE PATH
+    "Path to the SLEPc main directory.")
+set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
+
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")
 
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH

config/defaults.mk

@@ -125,6 +125,7 @@ MFEM_USE_GINKGO        = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
+MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
 MFEM_USE_CONDUIT       = NO
@@ -188,10 +189,12 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt
 
 # SUNDIALS library configuration
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
+# and modify cmake variables for hypre for sundials
 SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
 SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
 SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
- -lsundials_arkode -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol
+ -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
 
 ifeq ($(MFEM_USE_MPI),YES)
    SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
@@ -276,6 +279,20 @@ ifeq ($(PETSC_FOUND),YES)
       -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
 endif
 
+SLEPC_DIR := $(MFEM_DIR)/../slepc
+SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables
+SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,)
+SLEPC_INC_VAR = SLEPC_INCLUDE
+SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB
+ifeq ($(SLEPC_FOUND),YES)
+   SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
+   # Some additional external libraries might be defined in this file
+   -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
+   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
+   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
+      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
+endif
+
 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr

data/periodic-annulus-sector.geo

@@ -0,0 +1,37 @@
+SetFactory("OpenCASCADE");
+
+R1 = 1.0;
+R2 = 2.0;
+
+Point(1) = {0.0, 0, 0, 1.0};
+Point(2) = {R1, 0, 0, 1.0};
+Point(3) = {R2, 0, 0, 1.0};
+Point(4) = {R1*Cos(Pi/3), R1*Sin(Pi/3), 0, 1.0};
+Point(5) = {R2*Cos(Pi/3), R2*Sin(Pi/3), 0, 1.0};
+Line(1) = {2, 3};
+Line(2) = {4, 5};
+Circle(3) = {2, 1, 4};
+Circle(4) = {3, 1, 5};
+Curve Loop(5) = {1, 4, -2, -3};
+Plane Surface(1) = {5};
+
+Transfinite Curve{1} = 7;
+Transfinite Curve{2} = 7;
+Transfinite Curve{3} = 4;
+Transfinite Curve{4} = 10;
+
+// Set a rotation periodicity constraint:
+Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Pi/3};
+
+// Tag surfaces and volumes with positive integers
+Physical Curve(1) = {3};
+Physical Curve(2) = {4};
+Physical Curve(3) = {1};
+Physical Curve(4) = {2};
+Physical Surface(1) = {1};
+
+// Generate 2D mesh
+Mesh 2;
+Mesh.MshFileVersion = 2.2;
+
+Save "periodic-annulus-sector.msh";

data/periodic-annulus-sector.msh

@@ -0,0 +1,185 @@
+$MeshFormat
+2.2 0 8
+$EndMeshFormat
+$Nodes
+55
+1 1 0 0
+2 2 0 0
+3 0.5000000000000001 0.8660254037844386 0
+4 1 1.732050807568877 0
+5 1.166666666666667 0 0
+6 1.333333333333333 0 0
+7 1.5 0 0
+8 1.666666666666667 0 0
+9 1.833333333333333 0 0
+10 0.5833333333333335 1.010362971081845 0
+11 0.6666666666666667 1.154700538379251 0
+12 0.7500000000000002 1.299038105676658 0
+13 0.8333333333333335 1.443375672974064 0
+14 0.9166666666666669 1.587713240271471 0
+15 0.9396926207859085 0.3420201433256683 0
+16 0.7660444431189786 0.6427876096865386 0
+17 1.986476715483886 0.2321858282504602 0
+18 1.946089741159648 0.4612317414848793 0
+19 1.879385241571817 0.6840402866513365 0
+20 1.787265280646825 0.8975983604009234 0
+21 1.670975622825874 1.09901795614161 0
+22 1.532088886237958 1.285575219373077 0
+23 1.372483275737469 1.454747283146095 0
+24 1.194317183405575 1.604246385510085 0
+25 1.425989114816062 0.1915326920916892 0
+26 0.8788667344146573 1.13917645290495 0
+27 1.630372059110754 0.7154531062316609 0
+28 1.436395769298814 1.053728612482506 0
+29 1.081023776188756 0.6241293681829633 0
+30 1.168737372335971 1.428012728596308 0
+31 1.821063986059922 0.298149890497067 0
+32 1.234707097211386 0.3469796339295647 0
+33 1.377747393186519 0.6200150626754309 0
+34 1.457047681210906 0.3890895843559762 0
+35 0.917846726184522 0.8957978954532204 0
+36 1.218335619030348 0.9017812086952638 0
+37 1.066623110765233 1.061857005744772 0
+38 1.587029716281926 0.1355955181472859 0
+39 1.744445799211916 0.1441515753740107 0
+40 1.25 0.1443375672974065 0
+41 1.453660070628011 0.8435769396609902 0
+42 1.741367044061892 0.499612708014486 0
+43 1.30550638526547 1.257610469847477 0
+44 1.118213276932792 0.1666674689105279 0
+45 0.9109440214958271 1.306610291787315 0
+46 0.9970618258753989 1.438658589955562 0
+47 0.7499999999999998 1.010362971081845 0
+48 0.7034449005273667 0.8850673702175776 0
+49 1.605449512513618 0.9269067082200894 0
+50 1.561654019115059 0.5298592532912715 0
+51 1.229782222487711 1.096820457143683 0
+52 1.617066998712459 0.3090202662210922 0
+53 1.079645953234324 1.246963713711438 0
+54 1.877063966817811 0.1348974588243076 0
+55 1.055356609656722 1.558136350380461 0
+$EndNodes
+$Elements
+108
+1 1 2 3 1 1 5
+2 1 2 3 1 5 6
+3 1 2 3 1 6 7
+4 1 2 3 1 7 8
+5 1 2 3 1 8 9
+6 1 2 3 1 9 2
+7 1 2 4 2 3 10
+8 1 2 4 2 10 11
+9 1 2 4 2 11 12
+10 1 2 4 2 12 13
+11 1 2 4 2 13 14
+12 1 2 4 2 14 4
+13 1 2 1 3 1 15
+14 1 2 1 3 15 16
+15 1 2 1 3 16 3
+16 1 2 2 4 2 17
+17 1 2 2 4 17 18
+18 1 2 2 4 18 19
+19 1 2 2 4 19 20
+20 1 2 2 4 20 21
+21 1 2 2 4 21 22
+22 1 2 2 4 22 23
+23 1 2 2 4 23 24
+24 1 2 2 4 24 4
+25 2 2 1 1 32 40 25
+26 2 2 1 1 25 34 32
+27 2 2 1 1 33 41 36
+28 2 2 1 1 38 52 25
+29 2 2 1 1 33 36 29
+30 2 2 1 1 26 47 35
+31 2 2 1 1 35 37 26
+32 2 2 1 1 25 52 34
+33 2 2 1 1 32 44 40
+34 2 2 1 1 15 32 29
+35 2 2 1 1 15 29 16
+36 2 2 1 1 36 41 28
+37 2 2 1 1 32 33 29
+38 2 2 1 1 50 52 42
+39 2 2 1 1 32 34 33
+40 2 2 1 1 42 52 31
+41 2 2 1 1 43 53 51
+42 2 2 1 1 27 41 33
+43 2 2 1 1 26 53 45
+44 2 2 1 1 18 31 17
+45 2 2 1 1 29 35 16
+46 2 2 1 1 29 36 35
+47 2 2 1 1 24 30 23
+48 2 2 1 1 30 53 43
+49 2 2 1 1 17 54 2
+50 2 2 1 1 4 55 24
+51 2 2 1 1 28 51 36
+52 2 2 1 1 47 48 35
+53 2 2 1 1 36 37 35
+54 2 2 1 1 37 53 26
+55 2 2 1 1 22 28 21
+56 2 2 1 1 20 27 19
+57 2 2 1 1 33 50 27
+58 2 2 1 1 15 44 32
+59 2 2 1 1 18 42 31
+60 2 2 1 1 30 43 23
+61 2 2 1 1 35 48 16
+62 2 2 1 1 31 54 17
+63 2 2 1 1 9 39 8
+64 2 2 1 1 8 38 7
+65 2 2 1 1 7 25 6
+66 2 2 1 1 22 43 28
+67 2 2 1 1 23 43 22
+68 2 2 1 1 39 54 31
+69 2 2 1 1 19 42 18
+70 2 2 1 1 24 55 30
+71 2 2 1 1 27 42 19
+72 2 2 1 1 13 46 14
+73 2 2 1 1 51 53 37
+74 2 2 1 1 39 52 38
+75 2 2 1 1 6 40 5
+76 2 2 1 1 34 52 50
+77 2 2 1 1 12 45 13
+78 2 2 1 1 30 55 46
+79 2 2 1 1 10 47 11
+80 2 2 1 1 8 39 38
+81 2 2 1 1 28 49 21
+82 2 2 1 1 7 38 25
+83 2 2 1 1 41 49 28
+84 2 2 1 1 20 49 27
+85 2 2 1 1 11 26 12
+86 2 2 1 1 27 49 41
+87 2 2 1 1 31 52 39
+88 2 2 1 1 25 40 6
+89 2 2 1 1 2 54 9
+90 2 2 1 1 14 55 4
+91 2 2 1 1 45 53 46
+92 2 2 1 1 45 46 13
+93 2 2 1 1 5 44 1
+94 2 2 1 1 21 49 20
+95 2 2 1 1 46 53 30
+96 2 2 1 1 3 48 10
+97 2 2 1 1 34 50 33
+98 2 2 1 1 36 51 37
+99 2 2 1 1 26 45 12
+100 2 2 1 1 11 47 26
+101 2 2 1 1 27 50 42
+102 2 2 1 1 40 44 5
+103 2 2 1 1 43 51 28
+104 2 2 1 1 10 48 47
+105 2 2 1 1 9 54 39
+106 2 2 1 1 46 55 14
+107 2 2 1 1 1 44 15
+108 2 2 1 1 16 48 3
+$EndElements
+$Periodic
+1
+1 1 2
+Affine 0.5000000000000001 0.8660254037844386 0 0 -0.8660254037844386 0.5000000000000001 0 0 0 0 1 0 0 0 0 1
+7
+9 14
+6 11
+8 13
+5 10
+7 12
+2 4
+1 3
+$EndPeriodic

data/periodic-torus-sector.geo

@@ -0,0 +1,25 @@
+SetFactory("OpenCASCADE");
+
+R = 1.5;
+r = 0.5;
+
+Torus(1) = {0,0,0, R, r, Pi/3};
+
+pts() = PointsOf{ Volume{1}; };
+
+Characteristic Length{ pts() } = 0.25;
+
+// Set a rotation periodicity constraint:
+Periodic Surface{3} = {2} Rotate{{0,0,1}, {0,0,0}, Pi/3};
+
+// Tag surfaces and volumes with positive integers
+Physical Surface(1) = {1};
+Physical Surface(2) = {2};
+Physical Surface(3) = {3};
+Physical Volume(1) = {1};
+
+// Generate 3D mesh
+Mesh 3;
+
+Mesh.MshFileVersion = 2.2;
+Save "periodic-torus-sector.msh";

data/square-disc-nurbs-patch.mesh

@@ -0,0 +1,155 @@
+MFEM NURBS mesh v1.0
+
+dimension
+2
+
+elements
+5
+1 3 0 3 7 4
+1 3 3 2 6 7
+1 3 2 1 5 6
+1 3 1 0 4 5
+1 3 2 8 9 1
+
+boundary
+10
+1 1 0 3
+2 1 3 2
+2 1 1 0
+2 1 2 8
+2 1 9 1
+3 1 7 4
+3 1 6 7
+3 1 5 6
+3 1 4 5
+4 1 8 9
+
+edges
+15
+0 0 4
+0 3 7
+0 1 5
+0 2 6
+1 0 3
+1 4 7
+2 3 2
+2 7 6
+2 1 0
+2 5 4
+1 2 1
+1 6 5
+1 8 9
+3 2 8
+3 1 9
+
+vertices
+10
+
+patches
+
+knotvectors
+2
+2 3 0 0 0 1 1 1
+2 4 0 0 0 0.5 1 1 1
+
+dimension
+2
+
+controlpoints_cartesian
+-5 5 1
+-5 3.92523e-16 1
+-5 -5 1
+-2.47593 2.47593 1
+-4.95187 6.06429e-16 0.707107
+-2.47593 -2.47593 1
+-0.424264 0.424264 1
+-0.848528 1.03915e-16 0.707107
+-0.424264 -0.424264 1
+-0.353553 0.353553 1
+-0.707107 8.65956e-17 0.707107
+-0.353553 -0.353553 1
+
+knotvectors
+2
+2 3 0 0 0 1 1 1
+2 4 0 0 0 0.5 1 1 1
+
+dimension
+2
+
+controlpoints_cartesian
+-5 -5 1
+-1.17757e-15 -5 1
+5 -5 1
+-2.47593 -2.47593 1
+-9.09644e-16 -4.95187 0.707107
+2.47593 -2.47593 1
+-0.424264 -0.424264 1
+-1.55872e-16 -0.848528 0.707107
+0.424264 -0.424264 1
+-0.353553 -0.353553 1
+-1.29893e-16 -0.707107 0.707107
+0.353553 -0.353553 1
+
+knotvectors
+2
+2 3 0 0 0 1 1 1
+2 4 0 0 0 0.5 1 1 1
+
+dimension
+2
+
+controlpoints_cartesian
+5 -5 1
+5 -1.17757e-15 1
+5 5 1
+2.47593 -2.47593 1
+4.95187 -1.21286e-15 0.707107
+2.47593 2.47593 1
+0.424264 -0.424264 1
+0.848528 -2.07829e-16 0.707107
+0.424264 0.424264 1
+0.353553 -0.353553 1
+0.707107 -1.73191e-16 0.707107
+0.353553 0.353553 1
+
+knotvectors
+2
+2 3 0 0 0 1 1 1
+2 4 0 0 0 0.5 1 1 1
+
+dimension
+2
+
+controlpoints_cartesian
+5 5 1
+3.92523e-16 5 1
+-5 5 1
+2.47593 2.47593 1
+3.03215e-16 4.95187 0.707107
+-2.47593 2.47593 1
+0.424264 0.424264 1
+5.19574e-17 0.848528 0.707107
+-0.424264 0.424264 1
+0.353553 0.353553 1
+4.32978e-17 0.707107 0.707107
+-0.353553 0.353553 1
+
+knotvectors
+2
+2 3 0 0 0 1 1 1
+2 3 0 0 0 1 1 1
+
+dimension
+2
+
+controlpoints_cartesian
+5 -5 1
+10 -5 1
+15 -5 1
+5 0 1
+10 0 1
+15 0 1
+5 5 1
+10 5 1
+15 5 1

doc/CMakeLists.txt

@@ -16,36 +16,21 @@ if (DOXYGEN_FOUND)
   configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CodeDocumentation.conf.in
     ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf @ONLY)
 
-  if (UNIX)
-    # Only create symlinks if UNIX operating system
-    add_custom_target(doc
-      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
-      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
-      COMMAND ${CMAKE_COMMAND} -E create_symlink
-      ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
-      ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
-      BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
-      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-      COMMENT "Generating API documentation with Doxygen to CodeDocumentation.html"
-      VERBATIM)
 
-    add_custom_target(clean-doc
-      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
-      COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
-      COMMENT "Removing API documentation"
-      VERBATIM)
+  add_custom_target(doc
+    COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
+    COMMAND echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+    BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Generating API documentation with Doxygen to CodeDocumentation.html"
+    VERBATIM)
+
+  add_custom_target(clean-doc
+    COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+    COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/warnings.log
+    COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
+    COMMENT "Removing API documentation"
+    VERBATIM)
 
-  else (UNIX)
-    add_custom_target(doc
-      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
-      BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
-      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-      COMMENT "Generating API documentation with Doxygen to CodeDocumentation/html/index.html"
-      VERBATIM)
 
-    add_custom_target(clean-doc
-      COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
-      COMMENT "Removing API documentation"
-      VERBATIM)
-  endif (UNIX)
 endif (DOXYGEN_FOUND)

doc/CodeDocumentation.conf.in

@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "Finite element discretization library"
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.
 
-PROJECT_LOGO           =
+PROJECT_LOGO           = web/logo-small.png
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -746,7 +746,7 @@ WARN_FORMAT            = "$file:$line: $text"
 # messages should be written. If left blank the output is written to standard
 # error (stderr).
 
-WARN_LOGFILE           =
+WARN_LOGFILE           = warnings.log
 
 #---------------------------------------------------------------------------
 # Configuration options related to the input files
@@ -770,6 +770,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/examples/pumi \
                          @MFEM_SOURCE_DIR@/examples/hiop \
                          @MFEM_SOURCE_DIR@/examples/sundials \
+                         @MFEM_SOURCE_DIR@/miniapps/adjoint \			 
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
@@ -1470,7 +1471,7 @@ MATHJAX_FORMAT         = HTML-CSS
 # The default value is: http://cdn.mathjax.org/mathjax/latest.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_RELPATH        = https://cdn.llnl.gov/mathjax/2.7.2
+MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
 
 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example

doc/CodeDocumentation.dox

@@ -88,8 +88,8 @@ namespace mfem {
  * - <a class="el" href="ex24p_8cpp_source.html">Example 24p</a>: parallel mixed finite element spaces and interpolators
  * - <a class="el" href="ex25_8cpp_source.html">Example 25</a>: simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
  * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
- * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM 
- * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM 
+ * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM
+ * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
  *
  * <H4>SUNDIALS Examples</H4>
  * - Variants of Examples
@@ -101,6 +101,9 @@ namespace mfem {
  *   and
  *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>
  *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS classes\endlink
+ * - CVODES adjoint examples:
+ *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
+ *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>
  *
  * <H4>PETSc Examples</H4>
  * - Variants of Examples
@@ -140,7 +143,9 @@ namespace mfem {
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
+ * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
+
+* - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
  * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
  * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
@@ -149,6 +154,7 @@ namespace mfem {
  * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
  * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
  * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
+ * - <a class="el" href="trimmer_8cpp_source.html">Trimmer</a>: trim elements from existing meshes
  * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
  * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
  * - <a class="el" href="load-dc_8cpp_source.html">Load DC</a>: visualize fields saved via DataCollection classes
@@ -156,7 +162,6 @@ namespace mfem {
  * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
  * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
  * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
- * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *

doc/makefile

@@ -9,18 +9,25 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
+SHELL = /bin/bash
 MFEM_DIR ?= ..
 DOXYGEN_CONF = CodeDocumentation.conf
 
+
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
-	doxygen $(DOXYGEN_CONF)
-	rm -f CodeDocumentation.html
-	ln -s CodeDocumentation/html/index.html CodeDocumentation.html
+	@# Generate the html documentation
+	@doxygen $(DOXYGEN_CONF)
+	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
+	@cat warnings.log
+	@# Generate the log of undocumented methods
+	@( cat $(DOXYGEN_CONF) ; echo "GENERATE_HTML=NO" ; echo "EXTRACT_ALL=NO" ; echo "WARN_LOGFILE=undoc.log" ; echo "QUIET=YES" ) | doxygen - &> /dev/null
 
 clean:
 	rm -rf $(DOXYGEN_CONF) CodeDocumentation CodeDocumentation.html *~
+	rm -rf  undoc.log warnings.log
 
 $(DOXYGEN_CONF): $(MFEM_DIR)/doc/$(DOXYGEN_CONF).in
-	sed -e 's%@MFEM_SOURCE_DIR@%$(MFEM_DIR)%g' $(<) \
+	@sed -e 's%@MFEM_SOURCE_DIR@%$(MFEM_DIR)%g' $(<) \
 	  > $(DOXYGEN_CONF)
+

examples/CMakeLists.txt

@@ -91,7 +91,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
     add_test(NAME ${TEST_NAME}_ser
       COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
   else()
-    add_test(NAME ${TEST_NAME}_np=4
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
       $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
@@ -101,13 +101,22 @@ endforeach()
 
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=4
+  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
     COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
     ${MPIEXEC_PREFLAGS}
     $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
     ${MPIEXEC_POSTFLAGS})
 endif()
 
+# If SuperLU_DIST is enabled, add a test run that uses it.
+if (MFEM_USE_SUPERLU)
+  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS}
+    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
+    ${MPIEXEC_POSTFLAGS})
+endif()
+
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
   add_subdirectory(sundials)

examples/ex1.cpp

@@ -9,6 +9,8 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
+//               ex1 -m ../data/periodic-annulus-sector.msh
+//               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2
 //               ex1 -m ../data/square-disc-p3.mesh -o 3
 //               ex1 -m ../data/square-disc-nurbs.mesh -o -1
@@ -32,7 +34,8 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cuda
+//             * ex1 -pa -d ceed-cuda
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -100,8 +103,8 @@ int main(int argc, char *argv[])
    // 3. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
 
    // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
@@ -109,10 +112,10 @@ int main(int argc, char *argv[])
    //    elements.
    {
       int ref_levels =
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
+         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
       }
    }
 
@@ -120,66 +123,70 @@ int main(int argc, char *argv[])
    //    Lagrange finite elements of the specified order. If order < 1, we
    //    instead use an isoparametric/isogeometric space.
    FiniteElementCollection *fec;
+   bool delete_fec;
    if (order > 0)
    {
       fec = new H1_FECollection(order, dim);
+      delete_fec = true;
    }
-   else if (mesh->GetNodes())
+   else if (mesh.GetNodes())
    {
-      fec = mesh->GetNodes()->OwnFEC();
+      fec = mesh.GetNodes()->OwnFEC();
+      delete_fec = false;
       cout << "Using isoparametric FEs: " << fec->Name() << endl;
    }
    else
    {
       fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
    }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   FiniteElementSpace fespace(&mesh, fec);
    cout << "Number of finite element unknowns: "
-        << fespace->GetTrueVSize() << endl;
+        << fespace.GetTrueVSize() << endl;
 
    // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined by marking all
    //    the boundary attributes from the mesh as essential (Dirichlet) and
    //    converting them to a list of true dofs.
    Array<int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
+   if (mesh.bdr_attributes.Size())
    {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
       ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
    // 7. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
    //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(fespace);
+   LinearForm b(&fespace);
    ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
 
    // 8. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero,
    //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
+   GridFunction x(&fespace);
    x = 0.0;
 
    // 9. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   BilinearForm a(&fespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: eliminating boundary
    //     conditions, applying conforming constraints for non-conforming AMR,
    //     static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
 
    OperatorPtr A;
    Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
    cout << "Size of linear system: " << A->Height() << endl;
 
@@ -200,9 +207,9 @@ int main(int argc, char *argv[])
    }
    else // Jacobi preconditioning in partial assembly mode
    {
-      if (UsesTensorBasis(*fespace))
+      if (UsesTensorBasis(fespace))
       {
-         OperatorJacobiSmoother M(*a, ess_tdof_list);
+         OperatorJacobiSmoother M(a, ess_tdof_list);
          PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
       }
       else
@@ -212,13 +219,13 @@ int main(int argc, char *argv[])
    }
 
    // 12. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
+   a.RecoverFEMSolution(X, b, x);
 
    // 13. Save the refined mesh and the solution. This output can be viewed later
    //     using GLVis: "glvis -m refined.mesh -g sol.gf".
    ofstream mesh_ofs("refined.mesh");
    mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
+   mesh.Print(mesh_ofs);
    ofstream sol_ofs("sol.gf");
    sol_ofs.precision(8);
    x.Save(sol_ofs);
@@ -230,15 +237,14 @@ int main(int argc, char *argv[])
       int  visport   = 19916;
       socketstream sol_sock(vishost, visport);
       sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
+      sol_sock << "solution\n" << mesh << x << flush;
    }
 
    // 15. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete mesh;
+   if (delete_fec)
+   {
+      delete fec;
+   }
 
    return 0;
 }

examples/ex11p.cpp

@@ -8,6 +8,8 @@
 //               mpirun -np 4 ex11p -m ../data/escher.mesh
 //               mpirun -np 4 ex11p -m ../data/fichera.mesh
 //               mpirun -np 4 ex11p -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex11p -m ../data/periodic-annulus-sector.msh
+//               mpirun -np 4 ex11p -m ../data/periodic-torus-sector.msh -rs 1
 //               mpirun -np 4 ex11p -m ../data/toroid-wedge.mesh -o 2
 //               mpirun -np 4 ex11p -m ../data/square-disc-p2.vtk -o 2
 //               mpirun -np 4 ex11p -m ../data/square-disc-p3.mesh -o 3

examples/ex18.hpp

@@ -88,8 +88,6 @@ private:
    Vector funval2;
    Vector nor;
    Vector fluxN;
-   IntegrationPoint eip1;
-   IntegrationPoint eip2;
 
 public:
    FaceIntegrator(RiemannSolver &rsolver_, const int dim);
@@ -424,19 +422,16 @@ void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
 
-      Tr.Loc1.Transform(ip, eip1);
-      Tr.Loc2.Transform(ip, eip2);
+      Tr.SetAllIntPoints(&ip); // set face and element int. points
 
       // Calculate basis functions on both elements at the face
-      el1.CalcShape(eip1, shape1);
-      el2.CalcShape(eip2, shape2);
+      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, funval1);
       elfun2_mat.MultTranspose(shape2, funval2);
 
-      Tr.SetIntPoint(&ip);
-
       // Get the normal vector and the flux on the face
       CalcOrtho(Tr.Jacobian(), nor);
       const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);

examples/ex1p.cpp

@@ -9,6 +9,8 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
+//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
 //               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
 //               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
@@ -30,7 +32,8 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cuda
+//             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -109,8 +112,8 @@ int main(int argc, char *argv[])
    // 4. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
 
    // 5. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -118,23 +121,23 @@ int main(int argc, char *argv[])
    //    more than 10,000 elements.
    {
       int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
       for (int l = 0; l < ref_levels; l++)
       {
-         mesh->UniformRefinement();
+         mesh.UniformRefinement();
       }
    }
 
    // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
    {
       int par_ref_levels = 2;
       for (int l = 0; l < par_ref_levels; l++)
       {
-         pmesh->UniformRefinement();
+         pmesh.UniformRefinement();
       }
    }
 
@@ -142,13 +145,16 @@ int main(int argc, char *argv[])
    //    use continuous Lagrange finite elements of the specified order. If
    //    order < 1, we instead use an isoparametric/isogeometric space.
    FiniteElementCollection *fec;
+   bool delete_fec;
    if (order > 0)
    {
       fec = new H1_FECollection(order, dim);
+      delete_fec = true;
    }
-   else if (pmesh->GetNodes())
+   else if (pmesh.GetNodes())
    {
-      fec = pmesh->GetNodes()->OwnFEC();
+      fec = pmesh.GetNodes()->OwnFEC();
+      delete_fec = false;
       if (myid == 0)
       {
          cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -157,9 +163,10 @@ int main(int argc, char *argv[])
    else
    {
       fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
    }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
+   ParFiniteElementSpace fespace(&pmesh, fec);
+   HYPRE_Int size = fespace.GlobalTrueVSize();
    if (myid == 0)
    {
       cout << "Number of finite element unknowns: " << size << endl;
@@ -170,44 +177,44 @@ int main(int argc, char *argv[])
    //    by marking all the boundary attributes from the mesh as essential
    //    (Dirichlet) and converting them to a list of true dofs.
    Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
+   if (pmesh.bdr_attributes.Size())
    {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
       ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
    // 9. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system, which in this case is
    //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
+   ParLinearForm b(&fespace);
    ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
 
    // 10. Define the solution vector x as a parallel finite element grid function
    //     corresponding to fespace. Initialize x with initial guess of zero,
    //     which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
+   ParGridFunction x(&fespace);
    x = 0.0;
 
    // 11. Set up the parallel bilinear form a(.,.) on the finite element space
    //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   ParBilinearForm a(&fespace);
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 12. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
+   if (static_cond) { a.EnableStaticCondensation(); }
+   a.Assemble();
 
    OperatorPtr A;
    Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
@@ -215,9 +222,9 @@ int main(int argc, char *argv[])
    Solver *prec = NULL;
    if (pa)
    {
-      if (UsesTensorBasis(*fespace))
+      if (UsesTensorBasis(fespace))
       {
-         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
+         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
       }
    }
    else
@@ -235,7 +242,7 @@ int main(int argc, char *argv[])
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
+   a.RecoverFEMSolution(X, b, x);
 
    // 15. Save the refined mesh and the solution in parallel. This output can
    //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
@@ -246,7 +253,7 @@ int main(int argc, char *argv[])
 
       ofstream mesh_ofs(mesh_name.str().c_str());
       mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
+      pmesh.Print(mesh_ofs);
 
       ofstream sol_ofs(sol_name.str().c_str());
       sol_ofs.precision(8);
@@ -261,16 +268,14 @@ int main(int argc, char *argv[])
       socketstream sol_sock(vishost, visport);
       sol_sock << "parallel " << num_procs << " " << myid << "\n";
       sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
+      sol_sock << "solution\n" << pmesh << x << flush;
    }
 
    // 17. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
+   if (delete_fec)
+   {
+      delete fec;
+   }
    MPI_Finalize();
 
    return 0;

examples/ex22.cpp

@@ -13,6 +13,11 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
+// Device sample runs:
+//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
+//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
+//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
+//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -76,6 +81,8 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool herm_conv = true;
    bool exact_sol = true;
+   bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -106,6 +113,10 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -135,13 +146,18 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
    //    with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Refine the mesh to increase resolution. In this example we do
+   // 4. Refine the mesh to increase resolution. In this example we do
    //    'ref_levels' of uniform refinement where the user specifies
    //    the number of levels with the '-r' option.
    for (int l = 0; l < ref_levels; l++)
@@ -149,7 +165,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 4. Define a finite element space on the mesh. Here we use continuous
+   // 5. Define a finite element space on the mesh. Here we use continuous
    //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
    //    order.
    if (dim == 1 && prob != 0 )
@@ -171,7 +187,7 @@ int main(int argc, char *argv[])
    cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
         << endl;
 
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
    //    In this example, the boundary conditions are defined based on the type
    //    of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -183,12 +199,12 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system.
    ComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 7. Define the solution vector u as a complex finite element grid function
+   // 8. Define the solution vector u as a complex finite element grid function
    //    corresponding to fespace. Initialize u with initial guess of 1+0i or
    //    the exact solution if it is known.
    ComplexGridFunction u(fespace);
@@ -210,7 +226,6 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
-   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -263,7 +278,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 8. Set up the sesquilinear form a(.,.) on the finite element space
+   // 9. Set up the sesquilinear form a(.,.) on the finite element space
    //    corresponding to the damped harmonic oscillator operator of the
    //    appropriate type:
    //
@@ -282,6 +297,7 @@ int main(int argc, char *argv[])
    ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
 
    SesquilinearForm *a = new SesquilinearForm(fespace, conv);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    switch (prob)
    {
       case 0:
@@ -305,7 +321,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 8a. Set up the bilinear form for the preconditioner corresponding to the
+   // 9a. Set up the bilinear form for the preconditioner corresponding to the
    //     appropriate operator
    //
    //      0) A scalar H1 field
@@ -318,6 +334,8 @@ int main(int argc, char *argv[])
    //         -Grad(a Div) - omega^2 b + omega c
    //
    BilinearForm *pcOp = new BilinearForm(fespace);
+   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+
    switch (prob)
    {
       case 0:
@@ -338,9 +356,9 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 9. Assemble the form and the corresponding linear system, applying any
-   //    necessary transformations such as: assembly, eliminating boundary
-   //    conditions, conforming constraints for non-conforming AMR, etc.
+   // 10. Assemble the form and the corresponding linear system, applying any
+   //     necessary transformations such as: assembly, eliminating boundary
+   //     conditions, conforming constraints for non-conforming AMR, etc.
    a->Assemble();
    pcOp->Assemble();
 
@@ -348,28 +366,17 @@ int main(int argc, char *argv[])
    Vector B, U;
 
    a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
-   u = 0.0;
-   U = 0.0;
 
-   OperatorHandle PCOp;
-   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
+   cout << "Size of linear system: " << A->Width() << endl << endl;
 
-   {
-      ComplexSparseMatrix * Asp =
-         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
-
-      cout << "Size of linear system: "
-           << 2 * Asp->real().Width() << endl << endl;
-   }
-
-   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
+   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
    //     preconditioner based on the appropriate sparse smoother.
    {
       Array<int> blockOffsets;
       blockOffsets.SetSize(3);
       blockOffsets[0] = 0;
-      blockOffsets[1] = PCOp.Ptr()->Height();
-      blockOffsets[2] = PCOp.Ptr()->Height();
+      blockOffsets[1] = A->Height() / 2;
+      blockOffsets[2] = A->Height() / 2;
       blockOffsets.PartialSum();
 
       BlockDiagonalPreconditioner BDP(blockOffsets);
@@ -377,22 +384,31 @@ int main(int argc, char *argv[])
       Operator * pc_r = NULL;
       Operator * pc_i = NULL;
 
-      double s = 1.0;
-      switch (prob)
+      if (pa)
       {
-         case 0:
-            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-            break;
-         case 1:
-            pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
-            s = -1.0;
-            break;
-         case 2:
-            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-            break;
-
-         default: break; // This should be unreachable
+         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
       }
+      else
+      {
+         OperatorHandle PCOp;
+         pcOp->SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
+         switch (prob)
+         {
+            case 0:
+               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+               break;
+            case 1:
+               pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
+               break;
+            case 2:
+               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+               break;
+            default:
+               break; // This should be unreachable
+         }
+      }
+      double s = (prob != 1) ? 1.0 : -1.0;
       pc_i = new ScaledOperator(pc_r,
                                 (conv == ComplexOperator::HERMITIAN) ?
                                 s:-s);
@@ -410,9 +426,11 @@ int main(int argc, char *argv[])
       gmres.Mult(B, U);
    }
 
-   // 11. Recover the solution as a finite element grid function and compute the
+   // 12. Recover the solution as a finite element grid function and compute the
    //     errors if the exact solution is known.
    a->RecoverFEMSolution(U, b, u);
+   u.real().SyncMemory(u);
+   u.imag().SyncMemory(u);
 
    if (exact_sol)
    {
@@ -442,7 +460,7 @@ int main(int argc, char *argv[])
       cout << endl;
    }
 
-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -m mesh -g sol".
    {
       ofstream mesh_ofs("refined.mesh");
@@ -457,7 +475,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -516,7 +534,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 14. Free the used memory.
+   // 15. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex22p.cpp

@@ -13,6 +13,11 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
+// Device sample runs:
+//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
+//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
+//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
+//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -41,7 +46,6 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.
 
-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -84,6 +88,8 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool herm_conv = true;
    bool exact_sol = true;
+   bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -116,6 +122,10 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -152,19 +162,24 @@ int main(int argc, char *argv[])
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the serial mesh on all processors to increase the resolution.
+   // 5. Refine the serial mesh on all processors to increase the resolution.
    for (int l = 0; l < ser_ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -174,7 +189,7 @@ int main(int argc, char *argv[])
       pmesh->UniformRefinement();
    }
 
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
    //    the specified order.
    if (dim == 1 && prob != 0 )
@@ -202,7 +217,7 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 7. Determine the list of true (i.e. parallel conforming) essential
+   // 8. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined
    //    based on the type of mesh and the problem type.
    Array<int> ess_tdof_list;
@@ -214,14 +229,14 @@ int main(int argc, char *argv[])
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 8. Set up the parallel linear form b(.) which corresponds to the
+   // 9. Set up the parallel linear form b(.) which corresponds to the
    //    right-hand side of the FEM linear system.
    ParComplexLinearForm b(fespace, conv);
    b.Vector::operator=(0.0);
 
-   // 9. Define the solution vector u as a parallel complex finite element grid
-   //    function corresponding to fespace. Initialize u with initial guess of
-   //    1+0i or the exact solution if it is known.
+   // 10. Define the solution vector u as a parallel complex finite element grid
+   //     function corresponding to fespace. Initialize u with initial guess of
+   //     1+0i or the exact solution if it is known.
    ParComplexGridFunction u(fespace);
    ParComplexGridFunction * u_exact = NULL;
    if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -241,7 +256,6 @@ int main(int argc, char *argv[])
    VectorConstantCoefficient zeroVecCoef(zeroVec);
    VectorConstantCoefficient oneVecCoef(oneVec);
 
-   u = 0.0;
    switch (prob)
    {
       case 0:
@@ -296,7 +310,7 @@ int main(int argc, char *argv[])
                  << "window_title 'Exact: Imaginary Part'" << flush;
    }
 
-   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
    //     space corresponding to the damped harmonic oscillator operator of the
    //     appropriate type:
    //
@@ -315,6 +329,7 @@ int main(int argc, char *argv[])
    ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
 
    ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    switch (prob)
    {
       case 0:
@@ -338,7 +353,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 10a. Set up the parallel bilinear form for the preconditioner
+   // 11a. Set up the parallel bilinear form for the preconditioner
    //      corresponding to the appropriate operator
    //
    //      0) A scalar H1 field
@@ -351,6 +366,7 @@ int main(int argc, char *argv[])
    //         -Grad(a Div) - omega^2 b + omega c
    //
    ParBilinearForm *pcOp = new ParBilinearForm(fespace);
+   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    switch (prob)
    {
       case 0:
@@ -371,7 +387,7 @@ int main(int argc, char *argv[])
       default: break; // This should be unreachable
    }
 
-   // 11. Assemble the parallel bilinear form and the corresponding linear
+   // 12. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, etc.
@@ -382,30 +398,22 @@ int main(int argc, char *argv[])
    Vector B, U;
 
    a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
-   u = 0.0;
-   U = 0.0;
-
-   OperatorHandle PCOp;
-   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
 
    if (myid == 0)
    {
-      ComplexHypreParMatrix * Ahyp =
-         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
-
       cout << "Size of linear system: "
-           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
+           << 2 * fespace->GlobalTrueVSize() << endl << endl;
    }
 
-   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
    //     diagonal preconditioner based on the appropriate multigrid
    //     preconditioner from hypre.
    {
       Array<int> blockTrueOffsets;
       blockTrueOffsets.SetSize(3);
       blockTrueOffsets[0] = 0;
-      blockTrueOffsets[1] = PCOp.Ptr()->Height();
-      blockTrueOffsets[2] = PCOp.Ptr()->Height();
+      blockTrueOffsets[1] = A->Height() / 2;
+      blockTrueOffsets[2] = A->Height() / 2;
       blockTrueOffsets.PartialSum();
 
       BlockDiagonalPreconditioner BDP(blockTrueOffsets);
@@ -413,25 +421,34 @@ int main(int argc, char *argv[])
       Operator * pc_r = NULL;
       Operator * pc_i = NULL;
 
-      switch (prob)
+      if (pa)
       {
-         case 0:
-            pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
-            break;
-         case 1:
-            pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-            break;
-         case 2:
-            if (dim == 2 )
-            {
+         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
+      }
+      else
+      {
+         OperatorHandle PCOp;
+         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
+         switch (prob)
+         {
+            case 0:
+               pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
+               break;
+            case 1:
                pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-            }
-            else
-            {
-               pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
-            }
-            break;
-         default: break; // This should be unreachable
+               break;
+            case 2:
+               if (dim == 2 )
+               {
+                  pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
+               }
+               else
+               {
+                  pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
+               }
+               break;
+            default: break; // This should be unreachable
+         }
       }
       pc_i = new ScaledOperator(pc_r,
                                 (conv == ComplexOperator::HERMITIAN) ?
@@ -449,9 +466,11 @@ int main(int argc, char *argv[])
       fgmres.SetPrintLevel(1);
       fgmres.Mult(B, U);
    }
-   // 13. Recover the parallel grid function corresponding to U. This is the
+   // 14. Recover the parallel grid function corresponding to U. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(U, b, u);
+   u.real().SyncMemory(u);
+   u.imag().SyncMemory(u);
 
    if (exact_sol)
    {
@@ -484,7 +503,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 14. Save the refined mesh and the solution in parallel. This output can be
+   // 15. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
       ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -504,7 +523,7 @@ int main(int argc, char *argv[])
       u.imag().Save(sol_i_ofs);
    }
 
-   // 15. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -569,7 +588,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 16. Free the used memory.
+   // 17. Free the used memory.
    delete a;
    delete u_exact;
    delete pcOp;

examples/ex24.cpp

@@ -7,6 +7,7 @@
 //               ex24 -m ../data/beam-tet.mesh
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 1
+//               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               ex24 -m ../data/escher.mesh
 //               ex24 -m ../data/escher.mesh -o 2
 //               ex24 -m ../data/fichera.mesh
@@ -24,12 +25,13 @@
 //               ex24 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with two variants:
+//               spaces, with three variants:
 //
 //               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (div v, q) for v in H(div) tested against q in L_2
+//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
+//               3) (div v, q) for v in H(div) tested against q in L_2
 //
-//               Using different approaches, we project the gradient or
+//               Using different approaches, we project the gradient, curl, or
 //               divergence to the appropriate space.
 //
 //               We recommend viewing examples 1, 3, and 5 before viewing this
@@ -45,8 +47,11 @@ using namespace mfem;
 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
 double div_gradp_exact(const Vector &x);
+void v_exact(const Vector &x, Vector &v);
+void curlv_exact(const Vector &x, Vector &cv);
 
 int dim;
+double freq = 1.0, kappa;
 
 int main(int argc, char *argv[])
 {
@@ -65,7 +70,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
    args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H(Curl) or 1: H(Div)");
+                  "Choose between 0: grad, 1: curl, 2: div");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -83,6 +88,7 @@ int main(int argc, char *argv[])
       return 1;
    }
    args.PrintOptions(cout);
+   kappa = freq * M_PI;
 
    // 2. Enable hardware devices such as GPUs, and programming models such as
    //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -119,10 +125,15 @@ int main(int argc, char *argv[])
       trial_fec = new H1_FECollection(order, dim);
       test_fec = new ND_FECollection(order, dim);
    }
+   else if (prob == 1)
+   {
+      trial_fec = new ND_FECollection(order, dim);
+      test_fec = new RT_FECollection(order-1, dim);
+   }
    else
    {
-      trial_fec = new RT_FECollection(order - 1, dim);
-      test_fec = new L2_FECollection(order - 1, dim);
+      trial_fec = new RT_FECollection(order-1, dim);
+      test_fec = new L2_FECollection(order-1, dim);
    }
 
    FiniteElementSpace trial_fes(mesh, trial_fec);
@@ -136,6 +147,12 @@ int main(int argc, char *argv[])
       cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
       cout << "Number of H1 finite element unknowns: " << trial_size << endl;
    }
+   else if (prob == 1)
+   {
+      cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
+      cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
+           endl;
+   }
    else
    {
       cout << "Number of Raviart-Thomas finite element unknowns: "
@@ -150,12 +167,18 @@ int main(int argc, char *argv[])
    GridFunction x(&test_fes);
    FunctionCoefficient p_coef(p_exact);
    VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
+   VectorFunctionCoefficient v_coef(sdim, v_exact);
+   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
    FunctionCoefficient divgradp_coef(div_gradp_exact);
 
    if (prob == 0)
    {
       gftrial.ProjectCoefficient(p_coef);
    }
+   else if (prob == 1)
+   {
+      gftrial.ProjectCoefficient(v_coef);
+   }
    else
    {
       gftrial.ProjectCoefficient(gradp_coef);
@@ -179,6 +202,11 @@ int main(int argc, char *argv[])
       a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
       a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
    }
+   else if (prob == 1)
+   {
+      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
+   }
    else
    {
       a.AddDomainIntegrator(new MassIntegrator(one));
@@ -244,6 +272,10 @@ int main(int argc, char *argv[])
    {
       dlo.AddDomainInterpolator(new GradientInterpolator());
    }
+   else if (prob == 1)
+   {
+      dlo.AddDomainInterpolator(new CurlInterpolator());
+   }
    else
    {
       dlo.AddDomainInterpolator(new DivergenceInterpolator());
@@ -258,6 +290,10 @@ int main(int argc, char *argv[])
    {
       exact_proj.ProjectCoefficient(gradp_coef);
    }
+   else if (prob == 1)
+   {
+      exact_proj.ProjectCoefficient(curlv_coef);
+   }
    else
    {
       exact_proj.ProjectCoefficient(divgradp_coef);
@@ -276,10 +312,23 @@ int main(int argc, char *argv[])
       cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
            "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
       cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
-           "||_{L_2} = " << errInterp << '\n' << endl;
+           " ||_{L_2} = " << errInterp << '\n' << endl;
       cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
            "||_{L_2} = " << errProj << '\n' << endl;
    }
+   else if (prob == 1)
+   {
+      double errSol = x.ComputeL2Error(curlv_coef);
+      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
+      double errProj = exact_proj.ComputeL2Error(curlv_coef);
+
+      cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in H(div): "
+           "|| E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
+      cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
+           "||_{L_2} = " << errInterp << '\n' << endl;
+      cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
+           "||_{L_2} = " << errProj << '\n' << endl;
+   }
    else
    {
       int order_quad = max(2, 2*order+1);
@@ -295,7 +344,7 @@ int main(int argc, char *argv[])
 
       cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
            "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
+      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v "
            "||_{L_2} = " << errInterp << '\n' << endl;
       cout << " Projection f_h of exact div v in L_2: || f_h - div v "
            "||_{L_2} = " << errProj << '\n' << endl;
@@ -371,3 +420,33 @@ double div_gradp_exact(const Vector &x)
 
    return 0.0;
 }
+
+void v_exact(const Vector &x, Vector &v)
+{
+   if (dim == 3)
+   {
+      v(0) = sin(kappa * x(1));
+      v(1) = sin(kappa * x(2));
+      v(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      v(0) = sin(kappa * x(1));
+      v(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { v(2) = 0.0; }
+   }
+}
+
+void curlv_exact(const Vector &x, Vector &cv)
+{
+   if (dim == 3)
+   {
+      cv(0) = -kappa * cos(kappa * x(2));
+      cv(1) = -kappa * cos(kappa * x(0));
+      cv(2) = -kappa * cos(kappa * x(1));
+   }
+   else
+   {
+      cv = 0.0;
+   }
+}

examples/ex24p.cpp

@@ -6,7 +6,8 @@
 //               mpirun -np 4 ex24p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -p 1 -pa
+//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 1
+//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               mpirun -np 4 ex24p -m ../data/escher.mesh
 //               mpirun -np 4 ex24p -m ../data/escher.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/fichera.mesh
@@ -24,12 +25,13 @@
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with two variants:
+//               spaces, with three variants:
 //
 //               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (div v, q) for v in H(div) tested against q in L_2
+//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
+//               3) (div v, q) for v in H(div) tested against q in L_2
 //
-//               Using different approaches, we project the gradient or
+//               Using different approaches, we project the gradient, curl, or
 //               divergence to the appropriate space.
 //
 //               We recommend viewing examples 1, 3, and 5 before viewing this
@@ -45,8 +47,11 @@ using namespace mfem;
 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
 double div_gradp_exact(const Vector &x);
+void v_exact(const Vector &x, Vector &v);
+void curlv_exact(const Vector &x, Vector &cv);
 
 int dim;
+double freq = 1.0, kappa;
 
 int main(int argc, char *argv[])
 {
@@ -71,7 +76,7 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree).");
    args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H(Curl) or 1: H(Div)");
+                  "Choose between 0: grad, 1: curl, 2: div");
    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -96,6 +101,7 @@ int main(int argc, char *argv[])
    {
       args.PrintOptions(cout);
    }
+   kappa = freq * M_PI;
 
    // 3. Enable hardware devices such as GPUs, and programming models such as
    //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -147,10 +153,15 @@ int main(int argc, char *argv[])
       trial_fec = new H1_FECollection(order, dim);
       test_fec = new ND_FECollection(order, dim);
    }
+   else if (prob == 1)
+   {
+      trial_fec = new ND_FECollection(order, dim);
+      test_fec = new RT_FECollection(order-1, dim);
+   }
    else
    {
-      trial_fec = new RT_FECollection(order - 1, dim);
-      test_fec = new L2_FECollection(order - 1, dim);
+      trial_fec = new RT_FECollection(order-1, dim);
+      test_fec = new L2_FECollection(order-1, dim);
    }
 
    ParFiniteElementSpace trial_fes(pmesh, trial_fec);
@@ -166,6 +177,12 @@ int main(int argc, char *argv[])
          cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
          cout << "Number of H1 finite element unknowns: " << trial_size << endl;
       }
+      else if (prob == 1)
+      {
+         cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
+         cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
+              endl;
+      }
       else
       {
          cout << "Number of Raviart-Thomas finite element unknowns: "
@@ -181,12 +198,18 @@ int main(int argc, char *argv[])
    ParGridFunction x(&test_fes);
    FunctionCoefficient p_coef(p_exact);
    VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
+   VectorFunctionCoefficient v_coef(sdim, v_exact);
+   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
    FunctionCoefficient divgradp_coef(div_gradp_exact);
 
    if (prob == 0)
    {
       gftrial.ProjectCoefficient(p_coef);
    }
+   else if (prob == 1)
+   {
+      gftrial.ProjectCoefficient(v_coef);
+   }
    else
    {
       gftrial.ProjectCoefficient(gradp_coef);
@@ -210,6 +233,11 @@ int main(int argc, char *argv[])
       a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
       a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
    }
+   else if (prob == 1)
+   {
+      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
+   }
    else
    {
       a.AddDomainIntegrator(new MassIntegrator(one));
@@ -293,6 +321,10 @@ int main(int argc, char *argv[])
    {
       dlo.AddDomainInterpolator(new GradientInterpolator());
    }
+   else if (prob == 1)
+   {
+      dlo.AddDomainInterpolator(new CurlInterpolator());
+   }
    else
    {
       dlo.AddDomainInterpolator(new DivergenceInterpolator());
@@ -307,6 +339,10 @@ int main(int argc, char *argv[])
    {
       exact_proj.ProjectCoefficient(gradp_coef);
    }
+   else if (prob == 1)
+   {
+      exact_proj.ProjectCoefficient(curlv_coef);
+   }
    else
    {
       exact_proj.ProjectCoefficient(divgradp_coef);
@@ -324,14 +360,30 @@ int main(int argc, char *argv[])
 
       if (myid == 0)
       {
-         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
-              "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
-              "||_{L_2} = " << errInterp << '\n' << endl;
+         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl)"
+              ": || E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
+         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad"
+              " p ||_{L_2} = " << errInterp << '\n' << endl;
          cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
               "||_{L_2} = " << errProj << '\n' << endl;
       }
    }
+   else if (prob == 1)
+   {
+      double errSol = x.ComputeL2Error(curlv_coef);
+      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
+      double errProj = exact_proj.ComputeL2Error(curlv_coef);
+
+      if (myid == 0)
+      {
+         cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in "
+              "H(div): || E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
+         cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
+              "||_{L_2} = " << errInterp << '\n' << endl;
+         cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
+              "||_{L_2} = " << errProj << '\n' << endl;
+      }
+   }
    else
    {
       int order_quad = max(2, 2*order+1);
@@ -350,7 +402,7 @@ int main(int argc, char *argv[])
          cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
               "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
          cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
-              "||_{L_2} = " << errInterp << '\n' << endl;
+              " ||_{L_2} = " << errInterp << '\n' << endl;
          cout << " Projection f_h of exact div v in L_2: || f_h - div v "
               "||_{L_2} = " << errProj << '\n' << endl;
       }
@@ -436,3 +488,33 @@ double div_gradp_exact(const Vector &x)
 
    return 0.0;
 }
+
+void v_exact(const Vector &x, Vector &v)
+{
+   if (dim == 3)
+   {
+      v(0) = sin(kappa * x(1));
+      v(1) = sin(kappa * x(2));
+      v(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      v(0) = sin(kappa * x(1));
+      v(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { v(2) = 0.0; }
+   }
+}
+
+void curlv_exact(const Vector &x, Vector &cv)
+{
+   if (dim == 3)
+   {
+      cv(0) = -kappa * cos(kappa * x(2));
+      cv(1) = -kappa * cos(kappa * x(0));
+      cv(2) = -kappa * cos(kappa * x(1));
+   }
+   else
+   {
+      cv = 0.0;
+   }
+}

examples/ex25.cpp

@@ -10,6 +10,10 @@
 //               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
 //               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh
 //
+// Device sample runs:
+//               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh -pa -d cuda
+//               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh -pa -d cuda
+//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -82,24 +86,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLMatrixCoefficient : public MatrixCoefficient
+class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
-   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                              Vector &),
+                            CartesianPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+   virtual void Eval(Vector &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(height, width);
+      K.SetSize(vdim);
       (*Function)(transip, pml, K);
    }
 };
@@ -116,13 +120,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -153,6 +157,8 @@ int main(int argc, char *argv[])
    double freq = 5.0;
    bool herm_conv = true;
    bool visualization = 1;
+   bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -174,12 +180,21 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
 
    if (iprob > 4) { iprob = 4; }
    prob = (prob_type)iprob;
 
-   // 2. Setup the mesh
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Setup the mesh
    if (!mesh_file)
    {
       exact_known = true;
@@ -220,7 +235,7 @@ int main(int argc, char *argv[])
    // Setup PML length
    Array2D<double> length(dim, 2); length = 0.0;
 
-   // 3. Setup the Cartesian PML region.
+   // 4. Setup the Cartesian PML region.
    switch (prob)
    {
       case disc:
@@ -246,19 +261,19 @@ int main(int argc, char *argv[])
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
-   // 4. Refine the mesh to increase the resolution.
+   // 5. Refine the mesh to increase the resolution.
    for (int l = 0; l < ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 5. Reorient mesh in case of a tet mesh
+   // 6. Reorient mesh in case of a tet mesh
    mesh->ReorientTetMesh();
 
    // Set element attributes in order to distinguish elements in the PML region
    pml->SetAttributes(mesh);
 
-   // 6. Define a finite element space on the mesh. Here we use the Nedelec
+   // 7. Define a finite element space on the mesh. Here we use the Nedelec
    //    finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
@@ -266,7 +281,7 @@ int main(int argc, char *argv[])
 
    cout << "Number of finite element unknowns: " << size << endl;
 
-   // 7. Determine the list of true essential boundary dofs. In this example,
+   // 8. Determine the list of true essential boundary dofs. In this example,
    //    the boundary conditions are defined based on the specific mesh and the
    //    problem type.
    Array<int> ess_tdof_list;
@@ -308,12 +323,12 @@ int main(int argc, char *argv[])
    }
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 8. Setup Complex Operator convention
+   // 9. Setup Complex Operator convention
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
+   // 10. Set up the linear form b(.) which corresponds to the right-hand side of
+   //     the FEM linear system.
    VectorFunctionCoefficient f(dim, source);
    ComplexLinearForm b(fespace, conv);
    if (prob == load_src)
@@ -323,7 +338,7 @@ int main(int argc, char *argv[])
    b.Vector::operator=(0.0);
    b.Assemble();
 
-   // 10. Define the solution vector x as a complex finite element grid function
+   // 11. Define the solution vector x as a complex finite element grid function
    //     corresponding to fespace.
    ComplexGridFunction x(fespace);
    x = 0.0;
@@ -331,7 +346,7 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
    x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
 
-   // 11. Set up the sesquilinear form a(.,.)
+   // 12. Set up the sesquilinear form a(.,.)
    //
    //     In Comp
    //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -365,19 +380,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
-   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
+   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
-   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
+   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -385,30 +400,29 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                          new VectorFEMassIntegrator(restr_c2_Im));
 
-   // 12. Assemble the bilinear form and the corresponding linear system,
+   // 13. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: assembly, eliminating
    //     boundary conditions, applying conforming constraints for
    //     non-conforming AMR, etc.
-   a.Assemble();
+#ifndef MFEM_USE_SUITESPARSE
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+#endif
+   a.Assemble(0);
 
-   OperatorHandle Ah;
+   OperatorPtr A;
    Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
-
-   // 13. Transform to monolithic SparseMatrix
-   SparseMatrix *A = Ah.As<ComplexSparseMatrix>()->GetSystemMatrix();
-
-   cout << "Size of linear system: " << A->Height() << endl;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
    // 14. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
    {
-      UMFPackSolver  solver(*A);
-      solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      solver.Mult(B, X);
+      if (pa) { cout << "PA not available with MFEM_USE_SUITESPARSE" << endl; }
+      ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
+      csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      csolver.SetPrintLevel(1);
+      csolver.Mult(B, X);
    }
 #else
-
    // 14a. Set up the Bilinear form a(.,.) for the preconditioner
    //
    //    In Comp
@@ -424,45 +438,64 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
-      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
+      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
 
+      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
       prec.Assemble();
 
-      OperatorHandle PCOpAh;
-      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
       // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
-      //      preconditioner based on the Gauss-Seidel sparse smoother.
+      //      preconditioner based on the Gauss-Seidel or Jacobi sparse smoother.
       Array<int> offsets(3);
       offsets[0] = 0;
       offsets[1] = fespace->GetTrueVSize();
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      GSSmoother gs00(*PCOpAh.As<SparseMatrix>());
-      BlockDiagonalPreconditioner BlockGS(offsets);
-      ScaledOperator gs11(&gs00,
-                          (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
-      BlockGS.SetDiagonalBlock(0,&gs00);
-      BlockGS.SetDiagonalBlock(1,&gs11);
+      Operator *pc_r = nullptr;
+      Operator *pc_i = nullptr;
+      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      if (pa)
+      {
+         // Jacobi Smoother
+         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
+         ScaledOperator *d11 = new ScaledOperator(d00, s);
+         pc_r = d00;
+         pc_i = d11;
+      }
+      else
+      {
+         OperatorPtr PCOpAh;
+         prec.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
+         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
+         // Gauss-Seidel Smoother
+         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
+         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
+         pc_r = gs00;
+         pc_i = gs11;
+      }
+
+      BlockDiagonalPreconditioner BlockDP(offsets);
+      BlockDP.SetDiagonalBlock(0, pc_r);
+      BlockDP.SetDiagonalBlock(1, pc_i);
 
       GMRESSolver gmres;
       gmres.SetPrintLevel(1);
       gmres.SetKDim(200);
-      gmres.SetMaxIter(2000);
+      gmres.SetMaxIter(pa ? 5000 : 2000);
       gmres.SetRelTol(1e-5);
       gmres.SetAbsTol(0.0);
       gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockGS);
+      gmres.SetPreconditioner(BlockDP);
       gmres.Mult(B, X);
    }
 #endif
@@ -474,10 +507,8 @@ int main(int argc, char *argv[])
    // If exact is known compute the error
    if (exact_known)
    {
-      ComplexGridFunction x_gf(fespace);
       VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
       VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
-      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
       int order_quad = max(2, 2 * order + 1);
       const IntegrationRule *irs[Geometry::NumGeom];
       for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -573,7 +604,6 @@ int main(int argc, char *argv[])
    }
 
    // 18. Free the used memory.
-   delete A;
    delete pml;
    delete fespace;
    delete fec;
@@ -771,7 +801,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -782,14 +812,13 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -800,14 +829,13 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -818,14 +846,13 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -839,19 +866,18 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      M = (1.0 / det).real();
+      D = (1.0 / det).real();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -864,19 +890,18 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = (1.0 / det).imag();
+      D = (1.0 / det).imag();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -889,14 +914,13 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = abs(1.0 / det);
+      D = abs(1.0 / det);
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

examples/ex25p.cpp

@@ -10,6 +10,10 @@
 //               mpirun -np 4 ex25p -o 2 -f 8.0 -rs 2 -rp 2 -prob 4 -m ../data/inline-quad.mesh
 //               mpirun -np 4 ex25p -o 2 -f 2.0 -rs 1 -rp 1 -prob 4 -m ../data/inline-hex.mesh
 //
+// Device sample runs:
+//               mpirun -np 4 ex25p -o 1 -f 3.0 -rs 3 -rp 1 -prob 2 -pa -d cuda
+//               mpirun -np 4 ex25p -o 2 -f 1.0 -rs 1 -rp 1 -prob 3 -pa -d cuda
+//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -82,24 +86,24 @@ public:
 };
 
 // Class for returning the PML coefficients of the bilinear form
-class PMLMatrixCoefficient : public MatrixCoefficient
+class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
-   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                          DenseMatrix &),
-                        CartesianPML * pml_)
-      : MatrixCoefficient(dim), pml(pml_), Function(F)
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                              Vector &),
+                            CartesianPML * pml_)
+      : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
-   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+   virtual void Eval(Vector &K, ElementTransformation &T,
                      const IntegrationPoint &ip)
    {
       double x[3];
       Vector transip(x, 3);
       T.Transform(ip, transip);
-      K.SetSize(height, width);
+      K.SetSize(vdim);
       (*Function)(transip, pml, K);
    }
 };
@@ -116,13 +120,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -160,6 +164,8 @@ int main(int argc, char *argv[])
    double freq = 5.0;
    bool herm_conv = true;
    bool visualization = 1;
+   bool pa = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -183,12 +189,21 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
 
    if (iprob > 4) { iprob = 4; }
    prob = (prob_type)iprob;
 
-   // 3. Setup the (serial) mesh on all processors.
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 4. Setup the (serial) mesh on all processors.
    if (!mesh_file)
    {
       exact_known = true;
@@ -236,7 +251,7 @@ int main(int argc, char *argv[])
    // Setup PML length
    Array2D<double> length(dim, 2); length = 0.0;
 
-   // 4. Setup the Cartesian PML region.
+   // 5. Setup the Cartesian PML region.
    switch (prob)
    {
       case disc:
@@ -262,13 +277,13 @@ int main(int argc, char *argv[])
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 6. Refine the serial mesh on all processors to increase the resolution.
    for (int l = 0; l < ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh.
+   // 7. Define a parallel mesh by a partitioning of the serial mesh.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -278,13 +293,13 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6a. Reorient mesh in case of a tet mesh
+   // 7a. Reorient mesh in case of a tet mesh
    pmesh->ReorientTetMesh();
 
-   // 7. Set element attributes in order to distinguish elements in the PML
+   // 8. Set element attributes in order to distinguish elements in the PML
    pml->SetAttributes(pmesh);
 
-   // 8. Define a parallel finite element space on the parallel mesh. Here we
+   // 9. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -294,9 +309,9 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 9. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    based on the specific mesh and the problem type.
+   // 10. Determine the list of true (i.e. parallel conforming) essential
+   //     boundary dofs. In this example, the boundary conditions are defined
+   //     based on the specific mesh and the problem type.
    Array<int> ess_tdof_list;
    Array<int> ess_bdr;
    if (pmesh->bdr_attributes.Size())
@@ -336,11 +351,11 @@ int main(int argc, char *argv[])
    }
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 10. Setup Complex Operator convention
+   // 11. Setup Complex Operator convention
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 11. Set up the parallel linear form b(.) which corresponds to the
+   // 12. Set up the parallel linear form b(.) which corresponds to the
    //     right-hand side of the FEM linear system.
    VectorFunctionCoefficient f(dim, source);
    ParComplexLinearForm b(fespace, conv);
@@ -351,7 +366,7 @@ int main(int argc, char *argv[])
    b.Vector::operator=(0.0);
    b.Assemble();
 
-   // 12. Define the solution vector x as a parallel complex finite element grid
+   // 13. Define the solution vector x as a parallel complex finite element grid
    //     function corresponding to fespace.
    ParComplexGridFunction x(fespace);
    x = 0.0;
@@ -359,7 +374,7 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
    x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
 
-   // 13. Set up the parallel sesquilinear form a(.,.)
+   // 14. Set up the parallel sesquilinear form a(.,.)
    //
    //     In Comp
    //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -393,19 +408,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);
 
    int cdim = (dim == 2) ? 1 : dim;
-   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
-   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
+   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
 
-   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
-   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
+   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
 
    // Integrators inside the PML region
    a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -413,27 +428,25 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                          new VectorFEMassIntegrator(restr_c2_Im));
 
-   // 14. Assemble the parallel bilinear form and the corresponding linear
+   // 15. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, etc.
+#ifndef MFEM_USE_SUPERLU
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+#endif
    a.Assemble();
 
-   OperatorHandle Ah;
+   OperatorPtr Ah;
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
 
-   // 15. Transform to monolithic HypreParMatrix
-   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A->GetGlobalNumRows() << endl;
-   }
-
    // 16. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
    {
+      if (pa) { cout << "PA not available with MFEM_USE_SUPERLU" << endl; }
+      // Transform to monolithic HypreParMatrix
+      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
       SuperLURowLocMatrix SA(*A);
       SuperLUSolver superlu(MPI_COMM_WORLD);
       superlu.SetPrintStatistics(false);
@@ -441,9 +454,9 @@ int main(int argc, char *argv[])
       superlu.SetColumnPermutation(superlu::PARMETIS);
       superlu.SetOperator(SA);
       superlu.Mult(B, X);
+      delete A;
    }
 #else
-
    // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner
    //
    //    In Comp
@@ -459,22 +472,20 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));
 
-      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
-      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
+      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
 
-      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
 
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
 
+      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
       prec.Assemble();
 
-      OperatorHandle PCOpAh;
-      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
       // 16b. Define and apply a parallel GMRES solver for AU=B with a block
       //      diagonal preconditioner based on hypre's AMS preconditioner.
       Array<int> offsets(3);
@@ -483,21 +494,41 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      HypreAMS ams00(*PCOpAh.As<HypreParMatrix>(),fespace);
-      BlockDiagonalPreconditioner BlockAMS(offsets);
-      ScaledOperator ams11(&ams00,
-                           (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
-      BlockAMS.SetDiagonalBlock(0,&ams00);
-      BlockAMS.SetDiagonalBlock(1,&ams11);
+      Operator *pc_r = nullptr;
+      Operator *pc_i = nullptr;
+      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      if (pa)
+      {
+         // Jacobi Smoother
+         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
+         ScaledOperator *d11 = new ScaledOperator(d00, s);
+         pc_r = d00;
+         pc_i = d11;
+      }
+      else
+      {
+         OperatorPtr PCOpAh;
+         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
+         // Hypre AMS
+         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
+         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
+         pc_r = ams00;
+         pc_i = ams11;
+      }
+
+      BlockDiagonalPreconditioner BlockDP(offsets);
+      BlockDP.SetDiagonalBlock(0, pc_r);
+      BlockDP.SetDiagonalBlock(1, pc_i);
 
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetPrintLevel(1);
       gmres.SetKDim(200);
-      gmres.SetMaxIter(2000);
+      gmres.SetMaxIter(pa ? 5000 : 2000);
       gmres.SetRelTol(1e-5);
       gmres.SetAbsTol(0.0);
-      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockAMS);
+      gmres.SetOperator(*Ah);
+      gmres.SetPreconditioner(BlockDP);
       gmres.Mult(B, X);
    }
 #endif
@@ -509,10 +540,8 @@ int main(int argc, char *argv[])
    // If exact is known compute the error
    if (exact_known)
    {
-      ParComplexGridFunction x_gf(fespace);
       VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
       VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
-      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
       int order_quad = max(2, 2 * order + 1);
       const IntegrationRule *irs[Geometry::NumGeom];
       for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -629,7 +658,6 @@ int main(int argc, char *argv[])
    }
 
    // 20. Free the used memory.
-   delete A;
    delete pml;
    delete fespace;
    delete fec;
@@ -828,7 +856,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -839,14 +867,13 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).real();
+      D(i) = (det / pow(dxs[i], 2)).real();
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -857,14 +884,13 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = (det / pow(dxs[i], 2)).imag();
+      D(i) = (det / pow(dxs[i], 2)).imag();
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -875,14 +901,13 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
       det *= dxs[i];
    }
 
-   M = 0.0;
    for (int i = 0; i < dim; ++i)
    {
-      M(i, i) = abs(det / pow(dxs[i], 2));
+      D(i) = abs(det / pow(dxs[i], 2));
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -896,19 +921,18 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
    // in the 2D case the coefficient is scalar 1/det(J)
    if (dim == 2)
    {
-      M = (1.0 / det).real();
+      D = (1.0 / det).real();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).real();
+         D(i) = (pow(dxs[i], 2) / det).real();
       }
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -921,19 +945,18 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = (1.0 / det).imag();
+      D = (1.0 / det).imag();
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = (pow(dxs[i], 2) / det).imag();
+         D(i) = (pow(dxs[i], 2) / det).imag();
       }
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -946,14 +969,13 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 
    if (dim == 2)
    {
-      M = abs(1.0 / det);
+      D = abs(1.0 / det);
    }
    else
    {
-      M = 0.0;
       for (int i = 0; i < dim; ++i)
       {
-         M(i, i) = abs(pow(dxs[i], 2) / det);
+         D(i) = abs(pow(dxs[i], 2) / det);
       }
    }
 }

examples/ex4p.cpp

@@ -16,6 +16,7 @@
 //               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
+//               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb

examples/ex5.cpp

@@ -11,6 +11,12 @@
 //               ex5 -m ../data/escher.mesh
 //               ex5 -m ../data/fichera.mesh
 //
+// Device sample runs:
+//               ex5 -m ../data/star.mesh -pa -d cuda
+//               ex5 -m ../data/star.mesh -pa -d raja-cuda
+//               ex5 -m ../data/star.mesh -pa -d raja-omp
+//               ex5 -m ../data/beam-hex.mesh -pa -d cuda
+//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -50,6 +56,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    bool pa = false;
+   const char *device_config = "cpu";
    bool visualization = 1;
 
    OptionsParser args(argc, argv);
@@ -59,6 +66,8 @@ int main(int argc, char *argv[])
                   "Finite element order (polynomial degree).");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -70,13 +79,18 @@ int main(int argc, char *argv[])
    }
    args.PrintOptions(cout);
 
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   device.Print();
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
    //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
    //    the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 4. Refine the mesh to increase the resolution. In this example we do
    //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
    //    largest number that gives a final mesh with no more than 10,000
    //    elements.
@@ -89,7 +103,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 4. Define a finite element space on the mesh. Here we use the
+   // 5. Define a finite element space on the mesh. Here we use the
    //    Raviart-Thomas finite elements of the specified order.
    FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
    FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -97,7 +111,7 @@ int main(int argc, char *argv[])
    FiniteElementSpace *R_space = new FiniteElementSpace(mesh, hdiv_coll);
    FiniteElementSpace *W_space = new FiniteElementSpace(mesh, l2_coll);
 
-   // 5. Define the BlockStructure of the problem, i.e. define the array of
+   // 6. Define the BlockStructure of the problem, i.e. define the array of
    //    offsets for each variable. The last component of the Array is the sum
    //    of the dimensions of each block.
    Array<int> block_offsets(3); // number of variables + 1
@@ -112,7 +126,7 @@ int main(int argc, char *argv[])
    std::cout << "dim(R+W) = " << block_offsets.Last() << "\n";
    std::cout << "***********************************************************\n";
 
-   // 6. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 7. Define the coefficients, analytical solution, and rhs of the PDE.
    ConstantCoefficient k(1.0);
 
    VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -122,25 +136,28 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient ucoeff(dim, uFun_ex);
    FunctionCoefficient pcoeff(pFun_ex);
 
-   // 7. Allocate memory (x, rhs) for the analytical solution and the right hand
+   // 8. Allocate memory (x, rhs) for the analytical solution and the right hand
    //    side.  Define the GridFunction u,p for the finite element solution and
    //    linear forms fform and gform for the right hand side.  The data
    //    allocated by x and rhs are passed as a reference to the grid functions
    //    (u,p) and the linear forms (fform, gform).
-   BlockVector x(block_offsets), rhs(block_offsets);
+   MemoryType mt = device.GetMemoryType();
+   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
 
    LinearForm *fform(new LinearForm);
    fform->Update(R_space, rhs.GetBlock(0), 0);
    fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
    fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
    fform->Assemble();
+   fform->SyncAliasMemory(rhs);
 
    LinearForm *gform(new LinearForm);
    gform->Update(W_space, rhs.GetBlock(1), 0);
    gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
    gform->Assemble();
+   gform->SyncAliasMemory(rhs);
 
-   // 8. Assemble the finite element matrices for the Darcy operator
+   // 9. Assemble the finite element matrices for the Darcy operator
    //
    //                            D = [ M  B^T ]
    //                                [ B   0  ]
@@ -185,7 +202,7 @@ int main(int argc, char *argv[])
       darcyOp.SetBlock(1,0, &B);
    }
 
-   // 9. Construct the operators for preconditioner
+   // 10. Construct the operators for preconditioner
    //
    //                 P = [ diag(M)         0         ]
    //                     [  0       B diag(M)^-1 B^T ]
@@ -202,10 +219,11 @@ int main(int argc, char *argv[])
    if (pa)
    {
       mVarf->AssembleDiagonal(Md);
+      auto Md_host = Md.HostRead();
       Vector invMd(mVarf->Height());
       for (int i=0; i<mVarf->Height(); ++i)
       {
-         invMd(i) = 1.0 / Md(i);
+         invMd(i) = 1.0 / Md_host[i];
       }
 
       Vector BMBt_diag(bVarf->Height());
@@ -246,7 +264,7 @@ int main(int argc, char *argv[])
    darcyPrec.SetDiagonalBlock(0, invM);
    darcyPrec.SetDiagonalBlock(1, invS);
 
-   // 10. Solve the linear system with MINRES.
+   // 11. Solve the linear system with MINRES.
    //     Check the norm of the unpreconditioned residual.
    int maxIter(1000);
    double rtol(1.e-6);
@@ -263,6 +281,7 @@ int main(int argc, char *argv[])
    solver.SetPrintLevel(1);
    x = 0.0;
    solver.Mult(rhs, x);
+   if (device.IsEnabled()) { x.HostRead(); }
    chrono.Stop();
 
    if (solver.GetConverged())
@@ -273,7 +292,7 @@ int main(int argc, char *argv[])
                 << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
    std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
 
-   // 11. Create the grid functions u and p. Compute the L2 error norms.
+   // 12. Create the grid functions u and p. Compute the L2 error norms.
    GridFunction u, p;
    u.MakeRef(R_space, x.GetBlock(0), 0);
    p.MakeRef(W_space, x.GetBlock(1), 0);
@@ -293,7 +312,7 @@ int main(int argc, char *argv[])
    std::cout << "|| u_h - u_ex || / || u_ex || = " << err_u / norm_u << "\n";
    std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";
 
-   // 12. Save the mesh and the solution. This output can be viewed later using
+   // 13. Save the mesh and the solution. This output can be viewed later using
    //     GLVis: "glvis -m ex5.mesh -g sol_u.gf" or "glvis -m ex5.mesh -g
    //     sol_p.gf".
    {
@@ -310,13 +329,13 @@ int main(int argc, char *argv[])
       p.Save(p_ofs);
    }
 
-   // 13. Save data in the VisIt format
+   // 14. Save data in the VisIt format
    VisItDataCollection visit_dc("Example5", mesh);
    visit_dc.RegisterField("velocity", &u);
    visit_dc.RegisterField("pressure", &p);
    visit_dc.Save();
 
-   // 14. Save data in the ParaView format
+   // 15. Save data in the ParaView format
    ParaViewDataCollection paraview_dc("Example5", mesh);
    paraview_dc.SetPrefixPath("ParaView");
    paraview_dc.SetLevelsOfDetail(order);
@@ -328,7 +347,7 @@ int main(int argc, char *argv[])
    paraview_dc.RegisterField("pressure",&p);
    paraview_dc.Save();
 
-   // 15. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -341,7 +360,7 @@ int main(int argc, char *argv[])
       p_sock << "solution\n" << *mesh << p << "window_title 'Pressure'" << endl;
    }
 
-   // 16. Free the used memory.
+   // 17. Free the used memory.
    delete fform;
    delete gform;
    delete invM;

examples/ex5p.cpp

@@ -11,6 +11,12 @@
 //               mpirun -np 4 ex5p -m ../data/escher.mesh
 //               mpirun -np 4 ex5p -m ../data/fichera.mesh
 //
+// Device sample runs:
+//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d cuda
+//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-cuda
+//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-omp
+//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa -d cuda
+//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -60,6 +66,7 @@ int main(int argc, char *argv[])
    int order = 1;
    bool par_format = false;
    bool pa = false;
+   const char *device_config = "cpu";
    bool visualization = 1;
    bool adios2 = false;
 
@@ -75,6 +82,8 @@ int main(int argc, char *argv[])
                   "Format to use when saving the results for VisIt.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -96,13 +105,18 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 10,000 elements, unless the user specifies it as input.
@@ -118,7 +132,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -131,7 +145,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Raviart-Thomas finite elements of the specified order.
    FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
    FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -151,7 +165,7 @@ int main(int argc, char *argv[])
       std::cout << "***********************************************************\n";
    }
 
-   // 7. Define the two BlockStructure of the problem.  block_offsets is used
+   // 8. Define the two BlockStructure of the problem.  block_offsets is used
    //    for Vector based on dof (like ParGridFunction or ParLinearForm),
    //    block_trueOffstes is used for Vector based on trueDof (HypreParVector
    //    for the rhs and solution of the linear system).  The offsets computed
@@ -168,7 +182,7 @@ int main(int argc, char *argv[])
    block_trueOffsets[2] = W_space->TrueVSize();
    block_trueOffsets.PartialSum();
 
-   // 8. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 9. Define the coefficients, analytical solution, and rhs of the PDE.
    ConstantCoefficient k(1.0);
 
    VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -178,25 +192,30 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient ucoeff(dim, uFun_ex);
    FunctionCoefficient pcoeff(pFun_ex);
 
-   // 9. Define the parallel grid function and parallel linear forms, solution
-   //    vector and rhs.
-   BlockVector x(block_offsets), rhs(block_offsets);
-   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
+   // 10. Define the parallel grid function and parallel linear forms, solution
+   //     vector and rhs.
+   MemoryType mt = device.GetMemoryType();
+   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
+   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);
 
    ParLinearForm *fform(new ParLinearForm);
    fform->Update(R_space, rhs.GetBlock(0), 0);
    fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
    fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
    fform->Assemble();
+   fform->SyncAliasMemory(rhs);
    fform->ParallelAssemble(trueRhs.GetBlock(0));
+   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);
 
    ParLinearForm *gform(new ParLinearForm);
    gform->Update(W_space, rhs.GetBlock(1), 0);
    gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
    gform->Assemble();
+   gform->SyncAliasMemory(rhs);
    gform->ParallelAssemble(trueRhs.GetBlock(1));
+   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);
 
-   // 10. Assemble the finite element matrices for the Darcy operator
+   // 11. Assemble the finite element matrices for the Darcy operator
    //
    //                            D = [ M  B^T ]
    //                                [ B   0  ]
@@ -249,7 +268,7 @@ int main(int argc, char *argv[])
       darcyOp->SetBlock(1,0, B);
    }
 
-   // 11. Construct the operators for preconditioner
+   // 12. Construct the operators for preconditioner
    //
    //                 P = [ diag(M)         0         ]
    //                     [  0       B diag(M)^-1 B^T ]
@@ -266,10 +285,11 @@ int main(int argc, char *argv[])
    {
       Md_PA.SetSize(R_space->GetTrueVSize());
       mVarf->AssembleDiagonal(Md_PA);
+      auto Md_host = Md_PA.HostRead();
       Vector invMd(Md_PA.Size());
       for (int i=0; i<Md_PA.Size(); ++i)
       {
-         invMd(i) = 1.0 / Md_PA(i);
+         invMd(i) = 1.0 / Md_host[i];
       }
 
       Vector BMBt_diag(W_space->GetTrueVSize());
@@ -302,7 +322,7 @@ int main(int argc, char *argv[])
    darcyPr->SetDiagonalBlock(0, invM);
    darcyPr->SetDiagonalBlock(1, invS);
 
-   // 12. Solve the linear system with MINRES.
+   // 13. Solve the linear system with MINRES.
    //     Check the norm of the unpreconditioned residual.
    int maxIter(pa ? 1000 : 500);
    double rtol(1.e-6);
@@ -319,6 +339,7 @@ int main(int argc, char *argv[])
    solver.SetPrintLevel(verbose);
    trueX = 0.0;
    solver.Mult(trueRhs, trueX);
+   if (device.IsEnabled()) { trueX.HostRead(); }
    chrono.Stop();
 
    if (verbose)
@@ -332,7 +353,7 @@ int main(int argc, char *argv[])
       std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
    }
 
-   // 13. Extract the parallel grid function corresponding to the finite element
+   // 14. Extract the parallel grid function corresponding to the finite element
    //     approximation X. This is the local solution on each processor. Compute
    //     L2 error norms.
    ParGridFunction *u(new ParGridFunction);
@@ -360,7 +381,7 @@ int main(int argc, char *argv[])
       std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";
    }
 
-   // 14. Save the refined mesh and the solution in parallel. This output can be
+   // 15. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol_*".
    {
       ostringstream mesh_name, u_name, p_name;
@@ -381,7 +402,7 @@ int main(int argc, char *argv[])
       p->Save(p_ofs);
    }
 
-   // 15. Save data in the VisIt format
+   // 16. Save data in the VisIt format
    VisItDataCollection visit_dc("Example5-Parallel", pmesh);
    visit_dc.RegisterField("velocity", u);
    visit_dc.RegisterField("pressure", p);
@@ -390,7 +411,7 @@ int main(int argc, char *argv[])
                       DataCollection::PARALLEL_FORMAT);
    visit_dc.Save();
 
-   // 16. Save data in the ParaView format
+   // 17. Save data in the ParaView format
    ParaViewDataCollection paraview_dc("Example5P", pmesh);
    paraview_dc.SetPrefixPath("ParaView");
    paraview_dc.SetLevelsOfDetail(order);
@@ -402,7 +423,7 @@ int main(int argc, char *argv[])
    paraview_dc.RegisterField("pressure",p);
    paraview_dc.Save();
 
-   // 17. Optionally output a BP (binary pack) file using ADIOS2. This can be
+   // 18. Optionally output a BP (binary pack) file using ADIOS2. This can be
    //     visualized with the ParaView VTX reader.
 #ifdef MFEM_USE_ADIOS2
    if (adios2)
@@ -422,7 +443,7 @@ int main(int argc, char *argv[])
    }
 #endif
 
-   // 18. Send the solution by socket to a GLVis server.
+   // 19. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -442,7 +463,7 @@ int main(int argc, char *argv[])
              << endl;
    }
 
-   // 19. Free the used memory.
+   // 20. Free the used memory.
    delete fform;
    delete gform;
    delete u;

examples/ex6.cpp

@@ -20,7 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//               * ex6 -pa -d ceed-cuda
+//             * ex6 -pa -d ceed-cuda
 //               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh

examples/ex6p.cpp

@@ -20,7 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//               * mpirun -np 4 ex6p -pa -d ceed-cuda
+//             * mpirun -np 4 ex6p -pa -d ceed-cuda
 //               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh

examples/ex9.cpp

@@ -20,8 +20,11 @@
 // Device sample runs:
 //    ex9 -pa
 //    ex9 -ea
+//    ex9 -fa
 //    ex9 -pa -m ../data/periodic-cube.mesh
 //    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
+//    ex9 -ea -m ../data/periodic-cube.mesh -d cuda
+//    ex9 -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -144,6 +147,7 @@ int main(int argc, char *argv[])
    int order = 3;
    bool pa = false;
    bool ea = false;
+   bool fa = false;
    const char *device_config = "cpu";
    int ode_solver_type = 4;
    double t_final = 10.0;
@@ -170,6 +174,8 @@ int main(int argc, char *argv[])
                   "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
                   "--no-element-assembly", "Enable Element Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -278,6 +284,11 @@ int main(int argc, char *argv[])
       m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
       k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
    }
+   else if (fa)
+   {
+      m.SetAssemblyLevel(AssemblyLevel::FULL);
+      k.SetAssemblyLevel(AssemblyLevel::FULL);
+   }
    m.AddDomainIntegrator(new MassIntegrator);
    k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
    k.AddInteriorFaceIntegrator(
@@ -437,21 +448,19 @@ int main(int argc, char *argv[])
 FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
    : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
-   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
-   bool ea = M.GetAssemblyLevel() == AssemblyLevel::ELEMENT;
    Array<int> ess_tdof_list;
-   if (pa || ea)
+   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
+   {
+      M_prec = new DSmoother(M.SpMat());
+      M_solver.SetOperator(M.SpMat());
+      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
+   }
+   else
    {
       M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
       M_solver.SetOperator(M);
       dg_solver = NULL;
    }
-   else
-   {
-      M_prec = new DSmoother(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-      M_solver.SetOperator(M.SpMat());
-   }
    M_solver.SetPreconditioner(*M_prec);
    M_solver.iterative_mode = false;
    M_solver.SetRelTol(1e-9);

examples/ex9p.cpp

@@ -16,12 +16,16 @@
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 2 -rp 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 3 -rp 2 -dt 0.0025 -tf 9 -vs 20
 //    mpirun -np 4 ex9p -m ../data/periodic-cube.mesh -p 0 -o 2 -rp 1 -dt 0.01 -tf 8
+//    mpirun -np 3 ex9p -m ../data/amr-hex.mesh -p 1 -rs 1 -rp 0 -dt 0.005 -tf 0.5
 //
 // Device sample runs:
 //    mpirun -np 4 ex9p -pa
 //    mpirun -np 4 ex9p -ea
+//    mpirun -np 4 ex9p -fa
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
+//    mpirun -np 4 ex9p -ea -m ../data/periodic-cube.mesh -d cuda
+//    mpirun -np 4 ex9p -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -163,6 +167,7 @@ int main(int argc, char *argv[])
    int order = 3;
    bool pa = false;
    bool ea = false;
+   bool fa = false;
    const char *device_config = "cpu";
    int ode_solver_type = 4;
    double t_final = 10.0;
@@ -192,6 +197,8 @@ int main(int argc, char *argv[])
                   "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
                   "--no-element-assembly", "Enable Element Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -328,6 +335,12 @@ int main(int argc, char *argv[])
       m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
       k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
    }
+   else if (fa)
+   {
+      m->SetAssemblyLevel(AssemblyLevel::FULL);
+      k->SetAssemblyLevel(AssemblyLevel::FULL);
+   }
+
    m->AddDomainIntegrator(new MassIntegrator);
    k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
    k->AddInteriorFaceIntegrator(
@@ -564,29 +577,21 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
      M_solver(_M.ParFESpace()->GetComm()),
      z(_M.Height())
 {
-   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
-   bool ea = _M.GetAssemblyLevel()==AssemblyLevel::ELEMENT;
-
-   if (pa || ea)
-   {
-      M.Reset(&_M, false);
-      K.Reset(&_K, false);
-   }
-   else
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
    {
       M.Reset(_M.ParallelAssemble(), true);
       K.Reset(_K.ParallelAssemble(), true);
    }
+   else
+   {
+      M.Reset(&_M, false);
+      K.Reset(&_K, false);
+   }
 
    M_solver.SetOperator(*M);
 
    Array<int> ess_tdof_list;
-   if (pa || ea)
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }
-   else
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
    {
       HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
       HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
@@ -595,6 +600,11 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
 
       dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
    }
+   else
+   {
+      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
+      dg_solver = NULL;
+   }
 
    M_solver.SetPreconditioner(*M_prec);
    M_solver.iterative_mode = false;

examples/makefile

@@ -114,6 +114,11 @@ ex11p-test-strumpack: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
 test-par-YES: ex11p-test-strumpack
 endif
+ifeq ($(MFEM_USE_SUPERLU),YES)
+ex11p-test-superlu: ex11p
+	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
+test-par-YES: ex11p-test-superlu
+endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/CMakeLists.txt

@@ -34,6 +34,15 @@ if (MFEM_USE_MPI)
     )
 endif()
 
+if (MFEM_USE_SLEPC)
+  list(APPEND PETSC_EXAMPLES_SRCS
+    ex11p.cpp
+    )
+  list(APPEND PETSC_RC_FILES
+    rc_ex11p_lobpcg rc_ex11p_gd
+    )
+endif()
+
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 
@@ -78,12 +87,22 @@ set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
+if (MFEM_USE_SLEPC)
+  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
+  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
+  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
+endif()
 
 # Add the tests: one test per command-line-variable.
-foreach(TEST_OPTIONS_VAR
-        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+set(TEST_OPTIONS_VARS
+    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+if (MFEM_USE_SLEPC)
+    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
+endif()
+
+foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
   string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
   string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
   string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)

examples/petsc/ex11p.cpp

@@ -0,0 +1,440 @@
+//                       MFEM Example 11 - Parallel Version
+//                              PETSc Modification
+//
+// Compile with: make ex11p
+//
+// Sample runs:  mpirun -np 4 ex11p -m ../../data/star.mesh
+//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_lobpcg
+//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_gd
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example demonstrates the use of the SLEPc eigensolver as an
+//               alternative to the LOBPCG eigenvalue solver. The shift and
+//               invert spectral transformation is used to help the convergence
+//               to the smaller eigenvalues. Alternative solver parameters can
+//               be passed in a file with "-slepcopts".
+//
+//               Reusing a single GLVis visualization window for multiple
+//               eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+#ifndef MFEM_USE_SLEPC
+#error This examples requires that MFEM is build with MFEM_USE_SLEPC=YES
+#endif
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 2;
+   int par_ref_levels = 1;
+   int order = 1;
+   int nev = 5;
+   int seed = 75;
+   bool slu_solver  = false;
+   bool sp_solver = false;
+   bool visualization = 1;
+   bool use_slepc = true;
+   const char *slepcrc_file = "";
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&seed, "-s", "--seed",
+                  "Random seed used to initialize LOBPCG.");
+#ifdef MFEM_USE_SUPERLU
+   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
+                  "--no-superlu", "Use the SuperLU Solver.");
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
+                  "--no-strumpack", "Use the STRUMPACK Solver.");
+#endif
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&use_slepc, "-useslepc","--useslepc","-no-slepc",
+                  "--no-slepc","Use or not SLEPc to solve the eigenvalue problem");
+   args.AddOption(&slepcrc_file, "-slepcopts", "--slepcopts",
+                  "SlepcOptions file to use.");
+   args.Parse();
+   if (slu_solver && sp_solver)
+   {
+      if (myid == 0)
+         cout << "WARNING: Both SuperLU and STRUMPACK have been selected,"
+              << " please choose either one." << endl
+              << "         Defaulting to SuperLU." << endl;
+      sp_solver = false;
+   }
+   // The command line options are also passed to the STRUMPACK
+   // solver. So do not exit if some options are not recognized.
+   if (!sp_solver)
+   {
+      if (!args.Good())
+      {
+         if (myid == 0)
+         {
+            args.PrintUsage(cout);
+         }
+         MPI_Finalize();
+         return 1;
+      }
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2b. We initialize SLEPc. This internally initializes PETSc as well.
+   MFEMInitializeSlepc(NULL,NULL,slepcrc_file,NULL);
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs).
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution (1 time by
+   //    default, or specified on the command line with -rp). Once the parallel
+   //    mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (pmesh->GetNodes())
+   {
+      fec = pmesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of unknowns: " << size << endl;
+   }
+
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (pmesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (pmesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   a->Finalize();
+
+   ParBilinearForm *m = new ParBilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   // shift the eigenvalue corresponding to eliminated dofs to a large value
+   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
+   m->Finalize();
+
+   PetscParMatrix *pA = NULL, *pM = NULL;
+   HypreParMatrix *A = NULL, *M = NULL;
+   Operator::Type tid =
+      !use_slepc ? Operator::Hypre_ParCSR : Operator::PETSC_MATAIJ;
+   OperatorHandle Ah(tid), Mh(tid);
+
+   a->ParallelAssemble(Ah);
+   if (!use_slepc) { Ah.Get(A); }
+   else { Ah.Get(pA); }
+   Ah.SetOperatorOwner(false);
+
+   m->ParallelAssemble(Mh);
+   if (!use_slepc) {Mh.Get(M); }
+   else {Mh.Get(pM); }
+   Mh.SetOperatorOwner(false);
+
+#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
+   Operator * Arow = NULL;
+#ifdef MFEM_USE_SUPERLU
+   if (slu_solver)
+   {
+      Arow = new SuperLURowLocMatrix(*A);
+   }
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   if (sp_solver)
+   {
+      Arow = new STRUMPACKRowLocMatrix(*A);
+   }
+#endif
+#endif
+
+   delete a;
+   delete m;
+
+   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
+   //    preconditioner for A to be used within the solver. Set the matrices
+   //    which define the generalized eigenproblem A x = lambda M x.
+   Solver * precond = NULL;
+   if (!use_slepc)
+   {
+      if (!slu_solver && !sp_solver)
+      {
+         HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
+         amg->SetPrintLevel(0);
+         precond = amg;
+      }
+      else
+      {
+#ifdef MFEM_USE_SUPERLU
+         if (slu_solver)
+         {
+            SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
+            superlu->SetPrintStatistics(false);
+            superlu->SetSymmetricPattern(true);
+            superlu->SetColumnPermutation(superlu::PARMETIS);
+            superlu->SetOperator(*Arow);
+            precond = superlu;
+         }
+#endif
+#ifdef MFEM_USE_STRUMPACK
+         if (sp_solver)
+         {
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            strumpack->SetPrintFactorStatistics(true);
+            strumpack->SetPrintSolveStatistics(false);
+            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+            strumpack->DisableMatching();
+            strumpack->SetOperator(*Arow);
+            strumpack->SetFromCommandLine();
+            precond = strumpack;
+         }
+#endif
+      }
+   }
+
+   HypreLOBPCG * lobpcg = NULL;
+   SlepcEigenSolver * slepc = NULL;
+   if (!use_slepc)
+   {
+
+      lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
+      lobpcg->SetNumModes(nev);
+      lobpcg->SetRandomSeed(seed);
+      lobpcg->SetPreconditioner(*precond);
+      lobpcg->SetMaxIter(200);
+      lobpcg->SetTol(1e-8);
+      lobpcg->SetPrecondUsageMode(1);
+      lobpcg->SetPrintLevel(1);
+      lobpcg->SetMassMatrix(*M);
+      lobpcg->SetOperator(*A);
+   }
+   else
+   {
+      slepc = new SlepcEigenSolver(MPI_COMM_WORLD);
+      slepc->SetNumModes(nev);
+      slepc->SetWhichEigenpairs(SlepcEigenSolver::TARGET_REAL);
+      slepc->SetTarget(0.0);
+      slepc->SetSpectralTransformation(SlepcEigenSolver::SHIFT_INVERT);
+      slepc->SetOperators(*pA,*pM);
+   }
+
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
+   Array<double> eigenvalues;
+   if (!use_slepc)
+   {
+      lobpcg->Solve();
+      lobpcg->GetEigenvalues(eigenvalues);
+   }
+   else
+   {
+      slepc->Solve();
+      eigenvalues.SetSize(nev);
+      for (int i=0; i<nev; i++)
+      {
+         slepc->GetEigenvalue(i,eigenvalues[i]);
+      }
+   }
+   Vector temp(fespace->GetTrueVSize());
+   ParGridFunction x(fespace);
+
+   // 10. Save the refined mesh and the modes in parallel. This output can be
+   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh->Print(mesh_ofs);
+
+      for (int i=0; i<nev; i++)
+      {
+         // convert eigenvector from HypreParVector to ParGridFunction
+         if (!use_slepc)
+         {
+            x = lobpcg->GetEigenvector(i);
+         }
+         else
+         {
+            slepc->GetEigenvector(i,temp);
+            x.Distribute(temp);
+
+         }
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
+                   << setfill('0') << setw(6) << myid;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 11. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         if ( myid == 0 )
+         {
+            cout << "Eigenmode " << i+1 << '/' << nev
+                 << ", Lambda = " << eigenvalues[i] << endl;
+         }
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         if (!use_slepc)
+         {
+            x = lobpcg->GetEigenvector(i);
+         }
+         else
+         {
+            slepc->GetEigenvector(i,temp);
+            x.Distribute(temp);
+         }
+
+         mode_sock << "parallel " << num_procs << " " << myid << "\n"
+                   << "solution\n" << *pmesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         if (myid == 0)
+         {
+            cout << "press (q)uit or (c)ontinue --> " << flush;
+            cin >> c;
+         }
+         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 12. Free the used memory.
+   if (!use_slepc)
+   {
+      delete lobpcg;
+   }
+   else
+   {
+      delete slepc;
+   }
+   delete precond;
+   delete M;
+   delete A;
+#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
+   delete Arow;
+#endif
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete pmesh;
+
+   // We finalize SLEPc
+   MFEMFinalizeSlepc();
+   MPI_Finalize();
+
+   return 0;
+}

examples/petsc/makefile

@@ -23,6 +23,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES =
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex9p ex10p
+ifeq ($(MFEM_USE_SLEPC),YES)
+   PAR_EXAMPLES += ex11p
+endif
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -87,6 +90,9 @@ EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p
 EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
 EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
 EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
+EX11_ARGS_SINV   := -m ../../data/star.mesh --useslepc
+EX11_ARGS_LOBPCG := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
+EX11_ARGS_GD     := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd
 
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
@@ -114,6 +120,12 @@ ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
+ifeq ($(MFEM_USE_SLEPC),YES)
+ex11p-test-par: ex11p
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
+endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

examples/petsc/rc_ex11p_gd

@@ -0,0 +1,6 @@
+# Options for the eigenvalue solver
+-eps_view
+-eps_converged_reason
+-eps_type gd
+# Options for the spectral transform
+-st_type precond

examples/petsc/rc_ex11p_lobpcg

@@ -0,0 +1,11 @@
+# Options for the eigenvalue solver
+-eps_monitor
+-eps_converged_reason
+-eps_view_values
+-eps_type lobpcg
+-eps_gen_hermitian
+-eps_smallest_real
+-eps_lobpcg_blocksize 5
+# Options for the spectral transform
+-st_type precond
+-st_pc_type gamg

fem/bilinearform.cpp

@@ -76,7 +76,7 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
    precompute_sparsity = 0;
    diag_policy = DIAG_KEEP;
 
-   assembly = AssemblyLevel::FULL;
+   assembly = AssemblyLevel::LEGACYFULL;
    batch = 1;
    ext = NULL;
 }
@@ -94,7 +94,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
    precompute_sparsity = ps;
    diag_policy = DIAG_KEEP;
 
-   assembly = AssemblyLevel::FULL;
+   assembly = AssemblyLevel::LEGACYFULL;
    batch = 1;
    ext = NULL;
 
@@ -121,9 +121,10 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
    assembly = assembly_level;
    switch (assembly)
    {
+      case AssemblyLevel::LEGACYFULL:
+         break;
       case AssemblyLevel::FULL:
-         // ext = new FABilinearFormExtension(this);
-         // Use the original BilinearForm implementation for now
+         ext = new FABilinearFormExtension(this);
          break;
       case AssemblyLevel::ELEMENT:
          ext = new EABilinearFormExtension(this);
@@ -143,7 +144,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 void BilinearForm::EnableStaticCondensation()
 {
    delete static_cond;
-   if (assembly != AssemblyLevel::FULL)
+   if (assembly != AssemblyLevel::LEGACYFULL)
    {
       static_cond = NULL;
       MFEM_WARNING("Static condensation not supported for this assembly level");
@@ -168,7 +169,7 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                        const Array<int> &ess_tdof_list)
 {
    delete hybridization;
-   if (assembly != AssemblyLevel::FULL)
+   if (assembly != AssemblyLevel::LEGACYFULL)
    {
       delete constr_integ;
       hybridization = NULL;
@@ -223,7 +224,7 @@ MatrixInverse * BilinearForm::Inverse() const
 
 void BilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::FULL)
+   if (assembly == AssemblyLevel::LEGACYFULL)
    {
       if (!static_cond) { mat->Finalize(skip_zeros); }
       if (mat_e) { mat_e->Finalize(skip_zeros); }
@@ -639,8 +640,7 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
    }
    else
    {
-      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
-                 "matrix and use SparseMatrix::GetDiag?");
+      mat->GetDiag(diag);
    }
 }
 
@@ -1083,7 +1083,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    mat = NULL;
    mat_e = NULL;
    extern_bfs = 0;
-   assembly = AssemblyLevel::FULL;
+   assembly = AssemblyLevel::LEGACYFULL;
    ext = NULL;
 }
 
@@ -1108,7 +1108,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
    bbfi_marker = mbf->bbfi_marker;
    btfbfi_marker = mbf->btfbfi_marker;
 
-   assembly = AssemblyLevel::FULL;
+   assembly = AssemblyLevel::LEGACYFULL;
    ext = NULL;
 }
 
@@ -1121,6 +1121,8 @@ void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
    assembly = assembly_level;
    switch (assembly)
    {
+      case AssemblyLevel::LEGACYFULL:
+         break;
       case AssemblyLevel::FULL:
          // ext = new FAMixedBilinearFormExtension(this);
          // Use the original BilinearForm implementation for now
@@ -1191,7 +1193,7 @@ void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,
 
 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::FULL)
+   if (assembly != AssemblyLevel::LEGACYFULL)
    {
       MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
       return NULL;
@@ -1204,7 +1206,7 @@ MatrixInverse * MixedBilinearForm::Inverse() const
 
 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::FULL)
+   if (assembly == AssemblyLevel::LEGACYFULL)
    {
       mat -> Finalize (skip_zeros);
    }
@@ -1481,7 +1483,7 @@ void MixedBilinearForm::AssembleDiagonal_ADAt(const Vector &D,
 
 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::FULL)
+   if (assembly != AssemblyLevel::LEGACYFULL)
    {
       MFEM_WARNING("Conforming assemble not supported for this assembly level!");
       return;

fem/bilinearform.hpp

@@ -25,12 +25,15 @@
 namespace mfem
 {
 
-/// Enumeration defining the assembly level for bilinear and nonlinear form
-/// classes derived from Operator.
+/** @brief Enumeration defining the assembly level for bilinear and nonlinear
+    form classes derived from Operator. */
 enum class AssemblyLevel
 {
-   /// Fully assembled form, i.e. a global sparse matrix in MFEM, Hypre or PETSC
-   /// format.
+   /// Legacy fully assembled form, i.e. a global sparse matrix in MFEM, Hypre
+   /// or PETSC format. This assembly is ALWAYS performed on the host.
+   LEGACYFULL = 0,
+   /// Fully assembled form, i.e. a global sparse matrix in MFEM format. This
+   /// assembly is compatible with device execution.
    FULL,
    /// Form assembled at element level, which computes and stores dense element
    /// matrices.
@@ -44,15 +47,19 @@ enum class AssemblyLevel
 };
 
 
-/** Class for bilinear form - "Matrix" with associated FE space and
-    BLFIntegrators. */
+/** @brief A "square matrix" operator for the associated FE space and
+    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
+    M. This class also supports other assembly levels specified via the
+    SetAssemblyLevel() function. */
 class BilinearForm : public Matrix
 {
 protected:
-   /// Sparse matrix to be associated with the form. Owned.
+   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
    SparseMatrix *mat;
 
-   /// Matrix used to eliminate b.c. Owned.
+   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
+        from the b.c.  Owned.
+       \f$ M + M_e = M_{original} \f$ */
    SparseMatrix *mat_e;
 
    /// FE space on which the form lives. Not owned.
@@ -62,12 +69,12 @@ protected:
    AssemblyLevel assembly;
    /// Element batch size used in the form action (1, 8, num_elems, etc.)
    int batch;
-   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
+   /** @brief Extension for supporting Full Assembly (FA), Element Assembly (EA),
        Partial Assembly (PA), or Matrix Free assembly (MF). */
    BilinearFormExtension *ext;
 
-   /// Indicates the Mesh::sequence corresponding to the current state of the
-   /// BilinearForm.
+   /** @brief Indicates the Mesh::sequence corresponding to the current state of
+       the BilinearForm. */
    long sequence;
 
    /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
@@ -115,7 +122,7 @@ protected:
       static_cond = NULL; hybridization = NULL;
       precompute_sparsity = 0;
       diag_policy = DIAG_KEEP;
-      assembly = AssemblyLevel::FULL;
+      assembly = AssemblyLevel::LEGACYFULL;
       batch = 1;
       ext = NULL;
    }
@@ -147,35 +154,43 @@ public:
    /// Get the size of the BilinearForm as a square matrix.
    int Size() const { return height; }
 
-   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
-   /** This method must be called before assembly. */
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::FULL  (default)
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
    void SetAssemblyLevel(AssemblyLevel assembly_level);
 
    /// Returns the assembly level
    AssemblyLevel GetAssemblyLevel() const { return assembly; }
 
-   /** Enable the use of static condensation. For details see the description
-       for class StaticCondensation in fem/staticcond.hpp This method should be
-       called before assembly. If the number of unknowns after static
+   /** @brief Enable the use of static condensation. For details see the
+       description for class StaticCondensation in fem/staticcond.hpp This method
+       should be called before assembly. If the number of unknowns after static
        condensation is not reduced, it is not enabled. */
    void EnableStaticCondensation();
 
-   /** Check if static condensation was actually enabled by a previous call to
-       EnableStaticCondensation(). */
+   /** @brief Check if static condensation was actually enabled by a previous
+       call to EnableStaticCondensation(). */
    bool StaticCondensationIsEnabled() const { return static_cond; }
 
    /// Return the trace FE space associated with static condensation.
    FiniteElementSpace *SCFESpace() const
    { return static_cond ? static_cond->GetTraceFESpace() : NULL; }
 
-   /** Enable hybridization; for details see the description for class
+   /// Enable hybridization.
+   /** For details see the description for class
        Hybridization in fem/hybridization.hpp. This method should be called
        before assembly. */
    void EnableHybridization(FiniteElementSpace *constr_space,
                             BilinearFormIntegrator *constr_integ,
                             const Array<int> &ess_tdof_list);
 
-   /** For scalar FE spaces, precompute the sparsity pattern of the matrix
+   /** @brief For scalar FE spaces, precompute the sparsity pattern of the matrix
        (assuming dense element matrices) based on the types of integrators
        present in the bilinear form. */
    void UsePrecomputedSparsity(int ps = 1) { precompute_sparsity = ps; }
@@ -194,15 +209,16 @@ public:
    /// Use the sparsity of @a A to allocate the internal SparseMatrix.
    void UseSparsity(SparseMatrix &A);
 
-   /** Pre-allocate the internal SparseMatrix before assembly. If the flag
-       'precompute sparsity' is set, the matrix is allocated in CSR format (i.e.
+   /// Pre-allocate the internal SparseMatrix before assembly.
+   /**  If the flag 'precompute sparsity'
+       is set, the matrix is allocated in CSR format (i.e.
        finalized) and the entries are initialized with zeros. */
    void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
 
-   /// Access all integrators added with AddDomainIntegrator().
+   /// Access all the integrators added with AddDomainIntegrator().
    Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
 
-   /// Access all integrators added with AddBoundaryIntegrator().
+   /// Access all the integrators added with AddBoundaryIntegrator().
    Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
    /** @brief Access all boundary markers added with AddBoundaryIntegrator().
        If no marker was specified when the integrator was added, the
@@ -219,64 +235,85 @@ public:
        corresponding pointer (to Array<int>) will be NULL. */
    Array<Array<int>*> *GetBFBFI_Marker() { return &bfbfi_marker; }
 
+   /// Returns a reference to: \f$ M_{ij} \f$
    const double &operator()(int i, int j) { return (*mat)(i,j); }
 
-   /// Returns reference to a_{ij}.
+   /// Returns a reference to: \f$ M_{ij} \f$
    virtual double &Elem(int i, int j);
 
-   /// Returns constant reference to a_{ij}.
+   /// Returns constant reference to: \f$ M_{ij} \f$
    virtual const double &Elem(int i, int j) const;
 
-   /// Matrix vector multiplication.
+   /// Matrix vector multiplication:  \f$ y = M x \f$
    virtual void Mult(const Vector &x, Vector &y) const;
 
+   /** @brief Matrix vector multiplication with the original uneliminated
+       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
+       \f$ y = M x + M_e x \f$ */
    void FullMult(const Vector &x, Vector &y) const
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
+   /// Add the matrix vector multiple to a vector:  \f$ y += a M x \f$
    virtual void AddMult(const Vector &x, Vector &y, const double a = 1.0) const
    { mat -> AddMult (x, y, a); }
 
+   /** @brief Add the original uneliminated matrix vector multiple to a vector.
+       The original matrix is \f$ M + Me \f$ so we have:
+       \f$ y += M x + M_e x \f$ */
    void FullAddMult(const Vector &x, Vector &y) const
    { mat->AddMult(x, y); mat_e->AddMult(x, y); }
 
+   /// Add the matrix transpose vector multiplication:  \f$ y += a M^T x \f$
    virtual void AddMultTranspose(const Vector & x, Vector & y,
                                  const double a = 1.0) const
    { mat->AddMultTranspose(x, y, a); }
 
+   /** @brief Add the original uneliminated matrix transpose vector
+       multiple to a vector. The original matrix is \f$ M + M_e \f$
+       so we have: \f$ y += M^T x + {M_e}^T x \f$ */
    void FullAddMultTranspose(const Vector & x, Vector & y) const
    { mat->AddMultTranspose(x, y); mat_e->AddMultTranspose(x, y); }
 
+   /// Matrix transpose vector multiplication:  \f$ y = M^T x \f$
    virtual void MultTranspose(const Vector & x, Vector & y) const
    { y = 0.0; AddMultTranspose (x, y); }
 
+   /// Compute \f$ y^T M x \f$
    double InnerProduct(const Vector &x, const Vector &y) const
    { return mat->InnerProduct (x, y); }
 
-   /// Returns a pointer to (approximation) of the matrix inverse.
+   /// Returns a pointer to (approximation) of the matrix inverse:  \f$ M^{-1} \f$
    virtual MatrixInverse *Inverse() const;
 
    /// Finalizes the matrix initialization.
    virtual void Finalize(int skip_zeros = 1);
 
-   /// Returns a reference to the sparse matrix
+   /// Returns a const reference to the sparse matrix.
    const SparseMatrix &SpMat() const
    {
       MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
       return *mat;
    }
+
+   /// Returns a reference to the sparse matrix:  \f$ M \f$
    SparseMatrix &SpMat()
    {
       MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
       return *mat;
    }
+
+   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
+        to it.  Used for transfering ownership. */
    SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }
 
-   /// Returns a reference to the sparse matrix of eliminated b.c.
+   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
    const SparseMatrix &SpMatElim() const
    {
       MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
       return *mat_e;
    }
+
+   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
    SparseMatrix &SpMatElim()
    {
       MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
@@ -311,6 +348,7 @@ public:
    void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                              Array<int> &bdr_marker);
 
+   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
    void operator=(const double a)
    {
       if (mat != NULL) { *mat = a; }
@@ -328,10 +366,10 @@ public:
        for an AMR mesh. */
    void AssembleDiagonal(Vector &diag) const;
 
-   /// Get the finite element space prolongation matrix
+   /// Get the finite element space prolongation operator.
    virtual const Operator *GetProlongation() const
    { return fes->GetConformingProlongation(); }
-   /// Get the finite element space restriction matrix
+   /// Get the finite element space restriction operator
    virtual const Operator *GetRestriction() const
    { return fes->GetConformingRestriction(); }
    /// Get the output finite element space prolongation matrix
@@ -491,10 +529,12 @@ public:
                                  double value);
 
    /// Eliminate the given @a vdofs. NOTE: here, @a vdofs is a list of DOFs.
+   /** In this case the eliminations are applied to the internal \f$ M \f$
+       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
    void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                        DiagonalPolicy dpolicy = DIAG_ONE);
 
-   /// Eliminate the given @a vdofs, storing the eliminated part internally.
+   /// Eliminate the given @a vdofs, storing the eliminated part internally in \f$ M_e \f$.
    /** This method works in conjunction with EliminateVDofsInRHS() and allows
        elimination of boundary conditions in multiple right-hand sides. In this
        method, @a vdofs is a list of DOFs. */
@@ -523,9 +563,11 @@ public:
    void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
                             Vector &b);
 
+   /// Compute inner product for full uneliminated matrix \f$ y^T M x + y^T M_e x \f$
    double FullInnerProduct(const Vector &x, const Vector &y) const
    { return mat->InnerProduct(x, y) + mat_e->InnerProduct(x, y); }
 
+   /// Update the @a FiniteElementSpace and delete all data associated with the old one.
    virtual void Update(FiniteElementSpace *nfes = NULL);
 
    /// (DEPRECATED) Return the FE space associated with the BilinearForm.
@@ -537,7 +579,13 @@ public:
    /// Read-only access to the associated FiniteElementSpace.
    const FiniteElementSpace *FESpace() const { return fes; }
 
-   /// Sets diagonal policy used upon construction of the linear system
+   /// Sets diagonal policy used upon construction of the linear system.
+   /** Policies include:
+
+       - DIAG_ZERO (Set the diagonal values to zero)
+       - DIAG_ONE  (Set the diagonal values to one)
+       - DIAG_KEEP (Keep the diagonal values)
+   */
    void SetDiagonalPolicy(DiagonalPolicy policy);
 
    /// Indicate that integrators are not owned by the BilinearForm
@@ -550,16 +598,16 @@ public:
 
 /**
    Class for assembling of bilinear forms `a(u,v)` defined on different
-   trial and test spaces. The assembled matrix `A` is such that
+   trial and test spaces. The assembled matrix `M` is such that
 
-       a(u,v) = V^t A U
+       a(u,v) = V^t M U
 
    where `U` and `V` are the vectors representing the functions `u` and `v`,
    respectively.  The first argument, `u`, of `a(,)` is in the trial space
    and the second argument, `v`, is in the test space. Thus,
 
-       # of rows of A = dimension of the test space and
-       # of cols of A = dimension of the trial space.
+       # of rows of M = dimension of the test space and
+       # of cols of M = dimension of the trial space.
 
    Both trial and test spaces should be defined on the same mesh.
 */
@@ -628,11 +676,15 @@ public:
                      FiniteElementSpace *te_fes,
                      MixedBilinearForm *mbf);
 
+   /// Returns a reference to: \f$ M_{ij} \f$
    virtual double &Elem(int i, int j);
 
+   /// Returns a reference to: \f$ M_{ij} \f$
    virtual const double &Elem(int i, int j) const;
 
+   /// Matrix multiplication: \f$ y = M x \f$
    virtual void Mult(const Vector & x, Vector & y) const;
+
    virtual void AddMult(const Vector & x, Vector & y,
                         const double a = 1.0) const;
 
@@ -642,6 +694,7 @@ public:
 
    virtual MatrixInverse *Inverse() const;
 
+   /// Finalizes the matrix initialization.
    virtual void Finalize(int skip_zeros = 1);
 
    /** Extract the associated matrix as SparseMatrix blocks. The number of
@@ -649,8 +702,14 @@ public:
        test and trial spaces, respectively. */
    void GetBlocks(Array2D<SparseMatrix *> &blocks) const;
 
+   /// Returns a const reference to the sparse matrix:  \f$ M \f$
    const SparseMatrix &SpMat() const { return *mat; }
+
+   /// Returns a reference to the sparse matrix:  \f$ M \f$
    SparseMatrix &SpMat() { return *mat; }
+
+   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
+        to it.  Used for transfering ownership. */
    SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }
 
    /// Adds a domain integrator. Assumes ownership of @a bfi.
@@ -697,6 +756,7 @@ public:
        corresponding pointer (to Array<int>) will be NULL. */
    Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }
 
+   /// Sets all sparse values of \f$ M \f$ to @a a.
    void operator=(const double a) { *mat = a; }
 
    /// Set the desired assembly level. The default is AssemblyLevel::FULL.

fem/bilinearform_ext.cpp

@@ -15,6 +15,7 @@
 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
 #include "libceed/ceed.hpp"
+#include "pgridfunc.hpp"
 
 namespace mfem
 {
@@ -292,7 +293,8 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
 
 // Data and methods for element-assembled bilinear forms
 EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
-   : PABilinearFormExtension(form)
+   : PABilinearFormExtension(form),
+     factorize_face_terms(form->FESpace()->IsDGSpace())
 {
 }
 
@@ -347,6 +349,17 @@ void EABilinearFormExtension::Assemble()
    {
       bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr);
    }
+
+   if (factorize_face_terms && int_face_restrict_lex)
+   {
+      auto restFint = dynamic_cast<const L2FaceRestriction&>(*int_face_restrict_lex);
+      restFint.AddFaceMatricesToElementMatrices(ea_data_int, ea_data);
+   }
+   if (factorize_face_terms && bdr_face_restrict_lex)
+   {
+      auto restFbdr = dynamic_cast<const L2FaceRestriction&>(*bdr_face_restrict_lex);
+      restFbdr.AddFaceMatricesToElementMatrices(ea_data_bdr, ea_data);
+   }
 }
 
 void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
@@ -399,24 +412,27 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          const int NDOFS = faceDofs;
          auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
          auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
+         if (!factorize_face_terms)
          {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
+            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
+            MFEM_FORALL(glob_j, nf_int*NDOFS,
             {
-               res += A_int(i, j, 0, f)*X(i, 0, f);
-            }
-            Y(j, 0, f) += res;
-            res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_int(i, j, 1, f)*X(i, 1, f);
-            }
-            Y(j, 1, f) += res;
-         });
+               const int f = glob_j/NDOFS;
+               const int j = glob_j%NDOFS;
+               double res = 0.0;
+               for (int i = 0; i < NDOFS; i++)
+               {
+                  res += A_int(i, j, 0, f)*X(i, 0, f);
+               }
+               Y(j, 0, f) += res;
+               res = 0.0;
+               for (int i = 0; i < NDOFS; i++)
+               {
+                  res += A_int(i, j, 1, f)*X(i, 1, f);
+               }
+               Y(j, 1, f) += res;
+            });
+         }
          auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
          MFEM_FORALL(glob_j, nf_int*NDOFS,
          {
@@ -443,7 +459,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    // Treatment of boundary faces
    Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
    const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
+   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
    {
       // Apply the Boundary Face Restriction
       bdr_face_restrict_lex->Mult(x, faceBdrX);
@@ -522,24 +538,27 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          const int NDOFS = faceDofs;
          auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
          auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
+         if (!factorize_face_terms)
          {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
+            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
+            MFEM_FORALL(glob_j, nf_int*NDOFS,
             {
-               res += A_int(j, i, 0, f)*X(i, 0, f);
-            }
-            Y(j, 0, f) += res;
-            res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_int(j, i, 1, f)*X(i, 1, f);
-            }
-            Y(j, 1, f) += res;
-         });
+               const int f = glob_j/NDOFS;
+               const int j = glob_j%NDOFS;
+               double res = 0.0;
+               for (int i = 0; i < NDOFS; i++)
+               {
+                  res += A_int(j, i, 0, f)*X(i, 0, f);
+               }
+               Y(j, 0, f) += res;
+               res = 0.0;
+               for (int i = 0; i < NDOFS; i++)
+               {
+                  res += A_int(j, i, 1, f)*X(i, 1, f);
+               }
+               Y(j, 1, f) += res;
+            });
+         }
          auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
          MFEM_FORALL(glob_j, nf_int*NDOFS,
          {
@@ -566,7 +585,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
    // Treatment of boundary faces
    Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
    const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
+   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
    {
       // Apply the Boundary Face Restriction
       bdr_face_restrict_lex->Mult(x, faceBdrX);
@@ -595,6 +614,139 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
    }
 }
 
+// Data and methods for fully-assembled bilinear forms
+FABilinearFormExtension::FABilinearFormExtension(BilinearForm *form)
+   : EABilinearFormExtension(form),
+     mat(form->FESpace()->GetVSize(),form->FESpace()->GetVSize(),0),
+     face_mat(form->FESpace()->GetVSize(),0,0),
+     use_face_mat(false)
+{
+#ifdef MFEM_USE_MPI
+   if ( ParFiniteElementSpace* pfes =
+           dynamic_cast<ParFiniteElementSpace*>(form->FESpace()) )
+   {
+      if (pfes->IsDGSpace())
+      {
+         use_face_mat = true;
+         pfes->ExchangeFaceNbrData();
+         face_mat.SetWidth(pfes->GetFaceNbrVSize());
+      }
+   }
+#endif
+}
+
+void FABilinearFormExtension::Assemble()
+{
+   EABilinearFormExtension::Assemble();
+   FiniteElementSpace &fes = *a->FESpace();
+   if (fes.IsDGSpace())
+   {
+      const L2ElementRestriction *restE =
+         static_cast<const L2ElementRestriction*>(elem_restrict);
+      const L2FaceRestriction *restF =
+         static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
+      // 1. Fill I
+      //  1.1 Increment with restE
+      restE->FillI(mat);
+      //  1.2 Increment with restF
+      if (restF) { restF->FillI(mat, face_mat); }
+      //  1.3 Sum the non-zeros in I
+      auto h_I = mat.HostReadWriteI();
+      int cpt = 0;
+      const int vd = fes.GetVDim();
+      const int ndofs = ne*elemDofs*vd;
+      for (int i = 0; i < ndofs; i++)
+      {
+         const int nnz = h_I[i];
+         h_I[i] = cpt;
+         cpt += nnz;
+      }
+      const int nnz = cpt;
+      h_I[ndofs] = nnz;
+      mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
+      mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
+      if (use_face_mat && restF)
+      {
+         auto h_I_face = face_mat.HostReadWriteI();
+         int cpt = 0;
+         for (int i = 0; i < ndofs; i++)
+         {
+            const int nnz = h_I_face[i];
+            h_I_face[i] = cpt;
+            cpt += nnz;
+         }
+         const int nnz_face = cpt;
+         h_I_face[ndofs] = nnz_face;
+         face_mat.GetMemoryJ().New(nnz_face,
+                                   face_mat.GetMemoryJ().GetMemoryType());
+         face_mat.GetMemoryData().New(nnz_face,
+                                      face_mat.GetMemoryData().GetMemoryType());
+      }
+      // 2. Fill J and Data
+      // 2.1 Fill J and Data with Elem ea_data
+      restE->FillJAndData(ea_data, mat);
+      // 2.2 Fill J and Data with Face ea_data_ext
+      if (restF) { restF->FillJAndData(ea_data_ext, mat, face_mat); }
+      // 2.3 Shift indirections in I back to original
+      auto I = mat.HostReadWriteI();
+      for (int i = ndofs; i > 0; i--)
+      {
+         I[i] = I[i-1];
+      }
+      I[0] = 0;
+      if (use_face_mat && restF)
+      {
+         auto I_face = face_mat.HostReadWriteI();
+         for (int i = ndofs; i > 0; i--)
+         {
+            I_face[i] = I_face[i-1];
+         }
+         I_face[0] = 0;
+      }
+   }
+   else // continuous Galerkin case
+   {
+      const ElementRestriction &rest =
+         static_cast<const ElementRestriction&>(*elem_restrict);
+      rest.FillSparseMatrix(ea_data, mat);
+   }
+}
+
+void FABilinearFormExtension::Mult(const Vector &x, Vector &y) const
+{
+   mat.Mult(x, y);
+#ifdef MFEM_USE_MPI
+   if (const ParFiniteElementSpace *pfes =
+          dynamic_cast<const ParFiniteElementSpace*>(testFes))
+   {
+      ParGridFunction x_gf;
+      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
+                   const_cast<Vector&>(x),0);
+      x_gf.ExchangeFaceNbrData();
+      Vector &shared_x = x_gf.FaceNbrData();
+      if (shared_x.Size()) { face_mat.AddMult(shared_x, y); }
+   }
+#endif
+}
+
+void FABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
+{
+   mat.MultTranspose(x, y);
+#ifdef MFEM_USE_MPI
+   if (const ParFiniteElementSpace *pfes =
+          dynamic_cast<const ParFiniteElementSpace*>(testFes))
+   {
+      ParGridFunction x_gf;
+      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
+                   const_cast<Vector&>(x),0);
+      x_gf.ExchangeFaceNbrData();
+      Vector &shared_x = x_gf.FaceNbrData();
+      if (shared_x.Size()) { face_mat.AddMultTranspose(shared_x, y); }
+   }
+#endif
+}
+
+
 MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
    : Operator(form->Height(), form->Width()), a(form)
 {

fem/bilinearform_ext.hpp

@@ -22,9 +22,12 @@ namespace mfem
 class BilinearForm;
 class MixedBilinearForm;
 
-
-/** @brief Class extending the BilinearForm class to support the different
-    AssemblyLevel%s. */
+/// Class extending the BilinearForm class to support different AssemblyLevels.
+/**  FA - Full Assembly
+     PA - Partial Assembly
+     EA - Element Assembly
+     MF - Matrix Free
+*/
 class BilinearFormExtension : public Operator
 {
 protected:
@@ -42,6 +45,7 @@ public:
    /// Get the finite element space restriction matrix
    virtual const Operator *GetRestriction() const;
 
+   /// Assemble at the level given for the BilinearFormExtension subclass
    virtual void Assemble() = 0;
 
    virtual void AssembleDiagonal(Vector &diag) const
@@ -58,26 +62,6 @@ public:
    virtual void Update() = 0;
 };
 
-/// Data and methods for fully-assembled bilinear forms
-class FABilinearFormExtension : public BilinearFormExtension
-{
-public:
-   FABilinearFormExtension(BilinearForm *form)
-      : BilinearFormExtension(form) { }
-
-   /// TODO
-   void Assemble() {}
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
-   void FormLinearSystem(const Array<int> &ess_tdof_list,
-                         Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) {}
-   void Mult(const Vector &x, Vector &y) const {}
-   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
-   ~FABilinearFormExtension() {}
-};
-
 /// Data and methods for partially-assembled bilinear forms
 class PABilinearFormExtension : public BilinearFormExtension
 {
@@ -114,10 +98,12 @@ class EABilinearFormExtension : public PABilinearFormExtension
 protected:
    int ne;
    int elemDofs;
+   // The element matrices are stored row major
    Vector ea_data;
    int nf_int, nf_bdr;
    int faceDofs;
    Vector ea_data_int, ea_data_ext, ea_data_bdr;
+   bool factorize_face_terms;
 
 public:
    EABilinearFormExtension(BilinearForm *form);
@@ -127,7 +113,24 @@ public:
    void MultTranspose(const Vector &x, Vector &y) const;
 };
 
-/// Data and methods for matrix-free bilinear forms
+/// Data and methods for fully-assembled bilinear forms
+class FABilinearFormExtension : public EABilinearFormExtension
+{
+private:
+   SparseMatrix mat;
+   /// face_mat handles parallelism for DG face terms.
+   SparseMatrix face_mat;
+   bool use_face_mat;
+
+public:
+   FABilinearFormExtension(BilinearForm *form);
+
+   void Assemble();
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
+};
+
+/// Data and methods for matrix-free bilinear forms NOT YET IMPLEMENTED.
 class MFBilinearFormExtension : public BilinearFormExtension
 {
 public:
@@ -147,8 +150,12 @@ public:
    ~MFBilinearFormExtension() {}
 };
 
-/** @brief Class extending the MixedBilinearForm class to support the different
-    AssemblyLevel%s. */
+/// Class extending the MixedBilinearForm class to support different AssemblyLevels.
+/**  FA - Full Assembly
+     PA - Partial Assembly
+     EA - Element Assembly
+     MF - Matrix Free
+*/
 class MixedBilinearFormExtension : public Operator
 {
 protected:

fem/bilininteg.cpp

@@ -926,11 +926,14 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Trans.Loc1.Transform(ip, eip);
+
+      // Set the integration point in the face and the neighboring element
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
       el1.CalcShape(eip, shape);
 
-      Trans.SetIntPoint(&ip);
       w = Trans.Weight() * ip.weight;
       if (Q)
       {
@@ -1519,6 +1522,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
    double w;
 
 #ifdef MFEM_THREAD_SAFE
+   Vector D;
    DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
    curlshape.SetSize(nd,dimc);
@@ -1526,6 +1530,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
    elmat.SetSize(nd);
    if (MQ) { M.SetSize(dimc); }
+   if (DQ) { D.SetSize(dimc); }
 
    const IntegrationRule *ir = IntRule;
    if (ir == NULL)
@@ -1569,6 +1574,12 @@ void CurlCurlIntegrator::AssembleElementMatrix
          Mult(curlshape_dFt, M, curlshape);
          AddMultABt(curlshape, curlshape_dFt, elmat);
       }
+      else if (DQ)
+      {
+         DQ->Eval(D, Trans, ip);
+         D *= w;
+         AddMultADAt(curlshape_dFt, D, elmat);
+      }
       else if (Q)
       {
          w *= Q->Eval(Trans, ip);
@@ -2571,15 +2582,16 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
-      Trans.Loc1.Transform(ip, eip1);
-      if (ndof2)
-      {
-         Trans.Loc2.Transform(ip, eip2);
-      }
-      el1.CalcShape(eip1, shape1);
 
-      Trans.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
+      el1.CalcShape(eip1, shape1);
 
       u->Eval(vu, *Trans.Elem1, eip1);
 
@@ -2727,10 +2739,15 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
 
-      Trans.Loc1.Transform(ip, eip1);
-      Trans.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
       if (dim == 1)
       {
          nor(0) = 2*eip1.x - 1.0;
@@ -2787,7 +2804,6 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
 
       if (ndof2)
       {
-         Trans.Loc2.Transform(ip, eip2);
          el2.CalcShape(eip2, shape2);
          el2.CalcDShape(eip2, dshape2);
          w = ip.weight/2/Trans.Elem2->Weight();
@@ -3005,9 +3021,14 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
    for (int pind = 0; pind < ir->GetNPoints(); ++pind)
    {
       const IntegrationPoint &ip = ir->IntPoint(pind);
-      IntegrationPoint eip1, eip2; // integration point in the reference space
-      Trans.Loc1.Transform(ip, eip1);
-      Trans.SetIntPoint(&ip);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
 
       el1.CalcShape(eip1, shape1);
       el1.CalcDShape(eip1, dshape1);
@@ -3027,7 +3048,6 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
       double w, wLM;
       if (ndofs2)
       {
-         Trans.Loc2.Transform(ip, eip2);
          el2.CalcShape(eip2, shape2);
          el2.CalcDShape(eip2, dshape2);
          CalcAdjugate(Trans.Elem2->Jacobian(), adjJ);
@@ -3165,17 +3185,22 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip1, eip2;
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+
+      // Access the neighboring elements' integration points
+      // Note: eip2 will only contain valid data if Elem2 exists
+      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
+      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+
       // Trace finite element shape function
-      Trans.SetIntPoint(&ip);
       trial_face_fe.CalcShape(ip, face_shape);
       // Side 1 finite element shape function
-      Trans.Loc1.Transform(ip, eip1);
       test_fe1.CalcShape(eip1, shape1);
       if (ndof2)
       {
          // Side 2 finite element shape function
-         Trans.Loc2.Transform(ip, eip2);
          test_fe2.CalcShape(eip2, shape2);
       }
       w = ip.weight;

fem/bilininteg.hpp

@@ -20,6 +20,13 @@
 namespace mfem
 {
 
+// Local maximum size of dofs and quads in 1D
+constexpr int HCURL_MAX_D1D = 5;
+constexpr int HCURL_MAX_Q1D = 6;
+
+constexpr int HDIV_MAX_D1D = 5;
+constexpr int HDIV_MAX_Q1D = 6;
+
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
@@ -199,6 +206,8 @@ public:
    virtual ~BilinearFormIntegrator() { }
 };
 
+/** Wraps a given @a BilinearFormIntegrator and transposes the resulting element
+    matrices. See for example ex9, ex9p. */
 class TransposeIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -1563,7 +1572,7 @@ public:
 };
 
 /** Class for integrating the bilinear form a(u,v) := (-V u, Grad v) in 2D or 3D
-    and where V is a vector coefficient, u is in H1 and v is in H1. */
+    and where V is a vector coefficient, u is in H1 or L2 and v is in H1. */
 class MixedScalarWeakDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1683,6 +1692,22 @@ protected:
    {
       trial_fe.CalcPhysCurlShape(Trans, shape);
    }
+
+   using BilinearFormIntegrator::AssemblePA;
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector&, Vector&) const;
+
+private:
+   // PA extension
+   Vector pa_data;
+   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
+   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
+   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
+   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
+   const GeometricFactors *geom;   ///< Not owned
+   int dim, ne, dofs1D, dofs1Dtest,quad1D, testType, trialType, coeffDim;
 };
 
 /** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 3D and
@@ -1722,6 +1747,20 @@ protected:
    {
       test_fe.CalcPhysCurlShape(Trans, shape);
    }
+
+   using BilinearFormIntegrator::AssemblePA;
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
+
+   virtual void AddMultPA(const Vector&, Vector&) const;
+
+private:
+   // PA extension
+   Vector pa_data;
+   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
+   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
+   const GeometricFactors *geom;   ///< Not owned
+   int dim, ne, dofs1D, quad1D, testType, trialType, coeffDim;
 };
 
 /** Class for integrating the bilinear form a(u,v) := - (Q u, grad v) in either
@@ -2001,6 +2040,7 @@ public:
    void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };
 
+/** Mass integrator (u, v) restricted to the boundary of a domain */
 class BoundaryMassIntegrator : public MassIntegrator
 {
 public:
@@ -2260,12 +2300,14 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
    Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
+   Vector D;
    DenseMatrix curlshape, curlshape_dFt, M;
    DenseMatrix vshape, projcurl;
 #endif
 
 protected:
    Coefficient *Q;
+   VectorCoefficient *DQ;
    MatrixCoefficient *MQ;
 
    // PA extension
@@ -2274,12 +2316,17 @@ protected:
    const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
+   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
-   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
    /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }
+   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
 
    /* Given a particular Finite Element, compute the
       element curl-curl matrix elmat */
@@ -2357,8 +2404,11 @@ protected:
    Vector pa_data;
    const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
    const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
+   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
+   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D, fetype;
+   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
+   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
    VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
@@ -2379,6 +2429,8 @@ public:
 
    using BilinearFormIntegrator::AssemblePA;
    virtual void AssemblePA(const FiniteElementSpace &fes);
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes);
    virtual void AddMultPA(const Vector &x, Vector &y) const;
    virtual void AssembleDiagonalPA(Vector& diag);
 };

fem/bilininteg_convection_ea.cpp

@@ -32,7 +32,7 @@ static void EAConvectionAssemble1D(const int NE,
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.Write(), D1D, D1D, NE);
+   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -54,7 +54,7 @@ static void EAConvectionAssemble1D(const int NE,
             {
                val += r_Bj[k1] * D(k1, e) * r_Gi[k1];
             }
-            A(i1, j1, e) = val;
+            A(i1, j1, e) += val;
          }
       }
    });
@@ -76,7 +76,7 @@ static void EAConvectionAssemble2D(const int NE,
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, NE);
-   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -121,7 +121,7 @@ static void EAConvectionAssemble2D(const int NE,
                                * r_B[k1][j1]* r_B[k2][j2];
                      }
                   }
-                  A(i1, i2, j1, j2, e) = val;
+                  A(i1, i2, j1, j2, e) += val;
                }
             }
          }
@@ -145,7 +145,7 @@ static void EAConvectionAssemble3D(const int NE,
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -191,7 +191,7 @@ static void EAConvectionAssemble3D(const int NE,
                               }
                            }
                         }
-                        A(i1, i2, i3, j1, j2, j3, e) = val;
+                        A(i1, i2, i3, j1, j2, j3, e) += val;
                      }
                   }
                }

fem/bilininteg_convection_pa.cpp

@@ -788,6 +788,20 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    {
       vel = cQ->GetVec();
    }
+   else if (VectorQuadratureFunctionCoefficient* cQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+
+      qFun.Read();
+      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
    else
    {
       vel.SetSize(dim * nq * ne);

fem/bilininteg_dgtrace_pa.cpp

@@ -167,6 +167,19 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       r.SetSize(1);
       r(0) = c_rho->constant;
    }
+   else if (QuadratureFunctionCoefficient* c_rho =
+               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
+   {
+      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
    else
    {
       r.SetSize(nq * nf);
@@ -200,6 +213,20 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
    {
       vel = c_u->GetVec();
    }
+   else if (VectorQuadratureFunctionCoefficient* c_u =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
+   {
+      // Assumed to be in lexicographical ordering
+      const QuadratureFunction &qFun = c_u->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
    else
    {
       vel.SetSize(dim * nq * nf);

fem/bilininteg_diffusion_ea.cpp

@@ -31,7 +31,7 @@ static void EADiffusionAssemble1D(const int NE,
    MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.Write(), D1D, D1D, NE);
+   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -53,7 +53,7 @@ static void EADiffusionAssemble1D(const int NE,
             {
                val += r_Gj[k1] * D(k1, e) * r_Gi[k1];
             }
-            A(i1, j1, e) = val;
+            A(i1, j1, e) += val;
          }
       }
    });
@@ -75,7 +75,7 @@ static void EADiffusionAssemble2D(const int NE,
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -120,7 +120,7 @@ static void EADiffusionAssemble2D(const int NE,
                                + gbi * D11 * gbj;
                      }
                   }
-                  A(i1, i2, j1, j2, e) = val;
+                  A(i1, i2, j1, j2, e) += val;
                }
             }
          }
@@ -144,7 +144,7 @@ static void EADiffusionAssemble3D(const int NE,
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 6, NE);
-   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -208,7 +208,7 @@ static void EADiffusionAssemble3D(const int NE,
                               }
                            }
                         }
-                        A(i1, i2, i3, j1, j2, j3, e) = val;
+                        A(i1, i2, i3, j1, j2, j3, e) += val;
                      }
                   }
                }

fem/bilininteg_diffusion_pa.cpp

@@ -296,6 +296,19 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
       coeff.SetSize(1);
       coeff(0) = cQ->constant;
    }
+   else if (QuadratureFunctionCoefficient* cQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == ne*nq,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
    else
    {
       coeff.SetSize(nq * ne);
@@ -1307,7 +1320,33 @@ static void PADiffusionApply3D(const int NE,
    });
 }
 
-// Shared memory PA Diffusion Apply 3D kernel
+// Half of B and G are stored in shared to get B, Bt, G and Gt.
+// Indices computation for SmemPADiffusionApply3D.
+static MFEM_HOST_DEVICE inline int qi(const int q, const int d, const int Q)
+{
+   return (q<=d) ? q : Q-1-q;
+}
+
+static MFEM_HOST_DEVICE inline int dj(const int q, const int d, const int D)
+{
+   return (q<=d) ? d : D-1-d;
+}
+
+static MFEM_HOST_DEVICE inline int qk(const int q, const int d, const int Q)
+{
+   return (q<=d) ? Q-1-q : q;
+}
+
+static MFEM_HOST_DEVICE inline int dl(const int q, const int d, const int D)
+{
+   return (q<=d) ? D-1-d : d;
+}
+
+static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
+{
+   return (q<=d) ? -1.0 : 1.0;
+}
+
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
                                    const Array<double> &b_,
@@ -1320,28 +1359,27 @@ static void SmemPADiffusionApply3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
+   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= M1D, "");
+   MFEM_VERIFY(Q1D <= M1Q, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, 6, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
    {
-      const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
-      MFEM_SHARED double sBG[2][MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) (sBG+0);
-      double (*G)[MD1] = (double (*)[MD1]) (sBG+1);
-      double (*Bt)[MQ1] = (double (*)[MQ1]) (sBG+0);
-      double (*Gt)[MQ1] = (double (*)[MQ1]) (sBG+1);
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      MFEM_SHARED double sBG[MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) sBG;
+      double (*G)[MD1] = (double (*)[MD1]) sBG;
+      double (*Bt)[MQ1] = (double (*)[MQ1]) sBG;
+      double (*Gt)[MQ1] = (double (*)[MQ1]) sBG;
       MFEM_SHARED double sm0[3][MDQ*MDQ*MDQ];
       MFEM_SHARED double sm1[3][MDQ*MDQ*MDQ];
       double (*X)[MD1][MD1]    = (double (*)[MD1][MD1]) (sm0+2);
@@ -1359,108 +1397,127 @@ static void SmemPADiffusionApply3D(const int NE,
       double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
       double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
       double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dx,x,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
             {
                X[dz][dy][dx] = x(dx,dy,dz,e);
             }
          }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-               G[q][d] = g(q,d);
-            }
+            const int i = qi(qx,dy,Q1D);
+            const int j = dj(qx,dy,D1D);
+            const int k = qk(qx,dy,Q1D);
+            const int l = dl(qx,dy,D1D);
+            B[i][j] = b(qx,dy);
+            G[k][l] = g(qx,dy) * sign(qx,dy);
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            double u[D1D], v[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = 0.0; }
+            MFEM_UNROLL(MD1)
+            for (int dx = 0; dx < D1D; ++dx)
             {
-               double u = 0.0;
-               double v = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double coords = X[dz][dy][dx];
-                  u += coords * B[qx][dx];
-                  v += coords * G[qx][dx];
-               }
-               DDQ0[dz][dy][qx] = u;
-               DDQ1[dz][dy][qx] = v;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DDQ1[dz][dy][qx] * B[qy][dy];
-                  v += DDQ0[dz][dy][qx] * G[qy][dy];
-                  w += DDQ0[dz][dy][qx] * B[qy][dy];
-               }
-               DQQ0[dz][qy][qx] = u;
-               DQQ1[dz][qy][qx] = v;
-               DQQ2[dz][qy][qx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
+               const int i = qi(qx,dx,Q1D);
+               const int j = dj(qx,dx,D1D);
+               const int k = qk(qx,dx,Q1D);
+               const int l = dl(qx,dx,D1D);
+               const double s = sign(qx,dx);
+               MFEM_UNROLL(MD1)
                for (int dz = 0; dz < D1D; ++dz)
                {
-                  u += DQQ0[dz][qy][qx] * B[qz][dz];
-                  v += DQQ1[dz][qy][qx] * B[qz][dz];
-                  w += DQQ2[dz][qy][qx] * G[qz][dz];
+                  const double coords = X[dz][dy][dx];
+                  u[dz] += coords * B[i][j];
+                  v[dz] += coords * G[k][l] * s;
                }
-               QQQ0[qz][qy][qx] = u;
-               QQQ1[qz][qy][qx] = v;
-               QQQ2[qz][qy][qx] = w;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ0[dz][dy][qx] = u[dz];
+               DDQ1[dz][dy][qx] = v[dz];
             }
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            double u[D1D], v[D1D], w[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = w[dz] = 0.0; }
+            MFEM_UNROLL(MD1)
+            for (int dy = 0; dy < D1D; ++dy)
             {
-               const int q = qx + ((qy*Q1D) + (qz*Q1D*Q1D));
-               const double O11 = d(q,0,e);
-               const double O12 = d(q,1,e);
-               const double O13 = d(q,2,e);
-               const double O22 = d(q,3,e);
-               const double O23 = d(q,4,e);
-               const double O33 = d(q,5,e);
-               const double gX = QQQ0[qz][qy][qx];
-               const double gY = QQQ1[qz][qy][qx];
-               const double gZ = QQQ2[qz][qy][qx];
+               const int i = qi(qy,dy,Q1D);
+               const int j = dj(qy,dy,D1D);
+               const int k = qk(qy,dy,Q1D);
+               const int l = dl(qy,dy,D1D);
+               const double s = sign(qy,dy);
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; dz++)
+               {
+                  u[dz] += DDQ1[dz][dy][qx] * B[i][j];
+                  v[dz] += DDQ0[dz][dy][qx] * G[k][l] * s;
+                  w[dz] += DDQ0[dz][dy][qx] * B[i][j];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ0[dz][qy][qx] = u[dz];
+               DQQ1[dz][qy][qx] = v[dz];
+               DQQ2[dz][qy][qx] = w[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[Q1D], v[Q1D], w[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++) { u[qz] = v[qz] = w[qz] = 0.0; }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; qz++)
+               {
+                  const int i = qi(qz,dz,Q1D);
+                  const int j = dj(qz,dz,D1D);
+                  const int k = qk(qz,dz,Q1D);
+                  const int l = dl(qz,dz,D1D);
+                  const double s = sign(qz,dz);
+                  u[qz] += DQQ0[dz][qy][qx] * B[i][j];
+                  v[qz] += DQQ1[dz][qy][qx] * B[i][j];
+                  w[qz] += DQQ2[dz][qy][qx] * G[k][l] * s;
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               const double O11 = d(qx,qy,qz,0,e);
+               const double O12 = d(qx,qy,qz,1,e);
+               const double O13 = d(qx,qy,qz,2,e);
+               const double O22 = d(qx,qy,qz,3,e);
+               const double O23 = d(qx,qy,qz,4,e);
+               const double O33 = d(qx,qy,qz,5,e);
+               const double gX = u[qz];
+               const double gY = v[qz];
+               const double gZ = w[qz];
                QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
                QQQ1[qz][qy][qx] = (O12*gX) + (O22*gY) + (O23*gZ);
                QQQ2[qz][qy][qx] = (O13*gX) + (O23*gY) + (O33*gZ);
@@ -1468,78 +1525,112 @@ static void SmemPADiffusionApply3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      if (tidz == 0)
+      MFEM_FOREACH_THREAD(d,y,D1D)
       {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD(q,x,Q1D)
          {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[d][q] = b(q,d);
-               Gt[d][q] = g(q,d);
-            }
+            const int i = qi(q,d,Q1D);
+            const int j = dj(q,d,D1D);
+            const int k = qk(q,d,Q1D);
+            const int l = dl(q,d,D1D);
+            Bt[j][i] = b(q,d);
+            Gt[l][k] = g(q,d) * sign(q,d);
          }
       }
       MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
       {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD(dx,x,D1D)
          {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            double u[Q1D], v[Q1D], w[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
+            MFEM_UNROLL(MQ1)
+            for (int qx = 0; qx < Q1D; ++qx)
             {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += QQQ0[qz][qy][qx] * Gt[dx][qx];
-                  v += QQQ1[qz][qy][qx] * Bt[dx][qx];
-                  w += QQQ2[qz][qy][qx] * Bt[dx][qx];
-               }
-               QQD0[qz][qy][dx] = u;
-               QQD1[qz][qy][dx] = v;
-               QQD2[qz][qy][dx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += QQD0[qz][qy][dx] * Bt[dy][qy];
-                  v += QQD1[qz][qy][dx] * Gt[dy][qy];
-                  w += QQD2[qz][qy][dx] * Bt[dy][qy];
-               }
-               QDD0[qz][dy][dx] = u;
-               QDD1[qz][dy][dx] = v;
-               QDD2[qz][dy][dx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
+               const int i = qi(qx,dx,Q1D);
+               const int j = dj(qx,dx,D1D);
+               const int k = qk(qx,dx,Q1D);
+               const int l = dl(qx,dx,D1D);
+               const double s = sign(qx,dx);
+               MFEM_UNROLL(MQ1)
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  u += QDD0[qz][dy][dx] * Bt[dz][qz];
-                  v += QDD1[qz][dy][dx] * Bt[dz][qz];
-                  w += QDD2[qz][dy][dx] * Gt[dz][qz];
+                  u[qz] += QQQ0[qz][qy][qx] * Gt[l][k] * s;
+                  v[qz] += QQQ1[qz][qy][qx] * Bt[j][i];
+                  w[qz] += QQQ2[qz][qy][qx] * Bt[j][i];
                }
-               y(dx,dy,dz,e) += (u + v + w);
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD0[qz][qy][dx] = u[qz];
+               QQD1[qz][qy][dx] = v[qz];
+               QQD2[qz][qy][dx] = w[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D], v[Q1D], w[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
+            MFEM_UNROLL(MQ1)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const int i = qi(qy,dy,Q1D);
+               const int j = dj(qy,dy,D1D);
+               const int k = qk(qy,dy,Q1D);
+               const int l = dl(qy,dy,D1D);
+               const double s = sign(qy,dy);
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD0[qz][qy][dx] * Bt[j][i];
+                  v[qz] += QQD1[qz][qy][dx] * Gt[l][k] * s;
+                  w[qz] += QQD2[qz][qy][dx] * Bt[j][i];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD0[qz][dy][dx] = u[qz];
+               QDD1[qz][dy][dx] = v[qz];
+               QDD2[qz][dy][dx] = w[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D], v[D1D], w[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = v[dz] = w[dz] = 0.0; }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  const int i = qi(qz,dz,Q1D);
+                  const int j = dj(qz,dz,D1D);
+                  const int k = qk(qz,dz,Q1D);
+                  const int l = dl(qz,dz,D1D);
+                  const double s = sign(qz,dz);
+                  u[dz] += QDD0[qz][dy][dx] * Bt[j][i];
+                  v[dz] += QDD1[qz][dy][dx] * Bt[j][i];
+                  w[dz] += QDD2[qz][dy][dx] * Gt[l][k] * s;
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               y(dx,dy,dz,e) += (u[dz] + v[dz] + w[dz]);
             }
          }
       }
@@ -1574,9 +1665,11 @@ static void PADiffusionApply(const int dim,
       MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
    }
 #endif // MFEM_USE_OCCA
+   const int ID = (D1D << 4 ) | Q1D;
+
    if (dim == 2)
    {
-      switch ((D1D << 4 ) | Q1D)
+      switch (ID)
       {
          case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
          case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
@@ -1589,9 +1682,10 @@ static void PADiffusionApply(const int dim,
          default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
       }
    }
-   else if (dim == 3)
+
+   if (dim == 3)
    {
-      switch ((D1D << 4 ) | Q1D)
+      switch (ID)
       {
          case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
          case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);

fem/bilininteg_hdiv.cpp

@@ -23,11 +23,6 @@ using namespace std;
 namespace mfem
 {
 
-// Local maximum size of dofs and quads in 1D
-constexpr int HDIV_MAX_D1D = 5;
-constexpr int HDIV_MAX_Q1D = 6;
-
-
 // PA H(div) Mass Assemble 2D kernel
 void PAHdivSetup2D(const int Q1D,
                    const int NE,
@@ -114,6 +109,8 @@ void PAHdivMassApply2D(const int D1D,
                        Vector &_y)
 {
    constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
 
    auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
@@ -238,6 +235,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
                                   Vector &_diag)
 {
    constexpr static int VDIM = 2;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
 
    auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
@@ -614,6 +612,8 @@ static void PADivDivApply2D(const int D1D,
                             Vector &_y)
 {
    constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
 
    auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
    auto Bot = Reshape(_Bot.Read(), D1D-1, Q1D);
@@ -977,6 +977,7 @@ static void PADivDivAssembleDiagonal2D(const int D1D,
                                        Vector &_diag)
 {
    constexpr static int VDIM = 2;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
 
    auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
    auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
@@ -1400,6 +1401,8 @@ static void PAHdivL2Apply2D(const int D1D,
                             Vector &_y)
 {
    constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
 
    auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
    auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
@@ -1666,6 +1669,8 @@ static void PAHdivL2ApplyTranspose2D(const int D1D,
                                      Vector &_y)
 {
    constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
 
    auto L2Bo = Reshape(_L2Bo.Read(), Q1D, L2D1D);
    auto Gct = Reshape(_Gct.Read(), D1D, Q1D);
@@ -1724,7 +1729,7 @@ static void PAHdivL2ApplyTranspose2D(const int D1D,
 
       for (int qy = 0; qy < Q1D; ++qy)
       {
-         double aX[HDIV_MAX_D1D];
+         double aX[MAX_D1D];
 
          int osc = 0;
          for (int c = 0; c < VDIM; ++c)  // loop over x, y components

fem/bilininteg_mass_ea.cpp

@@ -30,7 +30,7 @@ static void EAMassAssemble1D(const int NE,
    MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
-   auto M = Reshape(eadata.Write(), D1D, D1D, NE);
+   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -52,7 +52,7 @@ static void EAMassAssemble1D(const int NE,
             {
                val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
             }
-            M(i1, j1, e) = val;
+            M(i1, j1, e) += val;
          }
       }
    });
@@ -72,7 +72,7 @@ static void EAMassAssemble2D(const int NE,
    MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
-   auto M = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
+   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -114,7 +114,7 @@ static void EAMassAssemble2D(const int NE,
                                * s_D[k1][k2];
                      }
                   }
-                  M(i1, i2, j1, j2, e) = val;
+                  M(i1, i2, j1, j2, e) += val;
                }
             }
          }
@@ -136,7 +136,7 @@ static void EAMassAssemble3D(const int NE,
    MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
-   auto M = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -189,7 +189,7 @@ static void EAMassAssemble3D(const int NE,
                               }
                            }
                         }
-                        M(i1, i2, i3, j1, j2, j3, e) = val;
+                        M(i1, i2, i3, j1, j2, j3, e) += val;
                      }
                   }
                }

fem/bilininteg_mass_pa.cpp

@@ -41,6 +41,8 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
       InitCeedCoeff(Q, ptr);
       return CeedPAMassAssemble(fes, *ir, *ptr);
    }
+#else
+   MFEM_CONTRACT_VAR(force);
 #endif
    dim = mesh->Dimension();
    ne = fes.GetMesh()->GetNE();
@@ -62,6 +64,19 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
       coeff.SetSize(1);
       coeff(0) = cQ->constant;
    }
+   else if (QuadratureFunctionCoefficient* cQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
    else
    {
       coeff.SetSize(nq * ne);
@@ -639,6 +654,7 @@ static void SmemPAMassApply2D(const int NE,
                               const int d1d = 0,
                               const int q1d = 0)
 {
+   MFEM_CONTRACT_VAR(bt_);
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
@@ -902,6 +918,7 @@ static void SmemPAMassApply3D(const int NE,
                               const int d1d = 0,
                               const int q1d = 0)
 {
+   MFEM_CONTRACT_VAR(bt_);
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;

fem/bilininteg_transpose_ea.cpp

@@ -25,7 +25,7 @@ void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
    if (ne == 0) { return; }
    const int dofs = fes.GetFE(0)->GetDof();
    auto A = Reshape(ea_data_tmp.Write(), dofs, dofs, ne);
-   auto AT = Reshape(ea_data.Write(), dofs, dofs, ne);
+   auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
    MFEM_FORALL(e, ne,
    {
       for (int i = 0; i < dofs; i++)

fem/bilininteg_vectorfe.cpp

@@ -9,12 +9,14 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#include "../general/forall.hpp"
 #include "bilininteg.hpp"
 
 namespace mfem
 {
 
 void PAHcurlSetup2D(const int Q1D,
+                    const int coeffDim,
                     const int NE,
                     const Array<double> &w,
                     const Vector &j,
@@ -22,6 +24,7 @@ void PAHcurlSetup2D(const int Q1D,
                     Vector &op);
 
 void PAHcurlSetup3D(const int Q1D,
+                    const int coeffDim,
                     const int NE,
                     const Array<double> &w,
                     const Vector &j,
@@ -31,6 +34,7 @@ void PAHcurlSetup3D(const int Q1D,
 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                    const int Q1D,
                                    const int NE,
+                                   const bool symmetric,
                                    const Array<double> &_Bo,
                                    const Array<double> &_Bc,
                                    const Vector &_op,
@@ -39,6 +43,7 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
 void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const int Q1D,
                                    const int NE,
+                                   const bool symmetric,
                                    const Array<double> &_Bo,
                                    const Array<double> &_Bc,
                                    const Vector &_op,
@@ -47,6 +52,7 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
 void PAHcurlMassApply2D(const int D1D,
                         const int Q1D,
                         const int NE,
+                        const bool symmetric,
                         const Array<double> &_Bo,
                         const Array<double> &_Bc,
                         const Array<double> &_Bot,
@@ -58,6 +64,7 @@ void PAHcurlMassApply2D(const int D1D,
 void PAHcurlMassApply3D(const int D1D,
                         const int Q1D,
                         const int NE,
+                        const bool symmetric,
                         const Array<double> &_Bo,
                         const Array<double> &_Bc,
                         const Array<double> &_Bot,
@@ -140,20 +147,573 @@ void PAHdivMassApply3D(const int D1D,
                        const Vector &_x,
                        Vector &_y);
 
+void PAHcurlL2Setup(const int NQ,
+                    const int coeffDim,
+                    const int NE,
+                    const Array<double> &w,
+                    Vector &_coeff,
+                    Vector &op);
+
+// PA H(curl) x H(div) mass assemble 3D kernel, with factor
+// dF^{-1} C dF for a vector or matrix coefficient C.
+// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
+void PAHcurlHdivSetup3D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const bool transpose,
+                        const Array<double> &_w,
+                        const Vector &j,
+                        Vector &_coeff,
+                        Vector &op)
+{
+   const int NQ = Q1D*Q1D*Q1D;
+   const bool symmetric = (coeffDim != 9);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 9, NQ, NE);
+
+   const int i11 = 0;
+   const int i12 = transpose ? 3 : 1;
+   const int i13 = transpose ? 6 : 2;
+   const int i21 = transpose ? 1 : 3;
+   const int i22 = 4;
+   const int i23 = transpose ? 7 : 5;
+   const int i31 = transpose ? 2 : 6;
+   const int i32 = transpose ? 5 : 7;
+   const int i33 = 8;
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double detJ = J11 * (J22 * J33 - J32 * J23) -
+         /* */               J21 * (J12 * J33 - J32 * J13) +
+         /* */               J31 * (J12 * J23 - J22 * J13);
+         const double w_detJ = W[q] / detJ;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+         {
+            // First compute entries of R = MJ
+            const double M11 = (!symmetric) ? coeff(i11, q, e) : coeff(0, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M13 = (!symmetric) ? coeff(i13, q, e) : coeff(2, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(3, q, e);
+            const double M23 = (!symmetric) ? coeff(i23, q, e) : coeff(4, q, e);
+            const double M31 = (!symmetric) ? coeff(i31, q, e) : M13;
+            const double M32 = (!symmetric) ? coeff(i32, q, e) : M23;
+            const double M33 = (!symmetric) ? coeff(i33, q, e) : coeff(5, q, e);
+
+            const double R11 = M11*J11 + M12*J12 + M13*J13;
+            const double R12 = M11*J21 + M12*J22 + M13*J23;
+            const double R13 = M11*J31 + M12*J32 + M13*J33;
+            const double R21 = M21*J11 + M22*J12 + M23*J13;
+            const double R22 = M21*J21 + M22*J22 + M23*J23;
+            const double R23 = M21*J31 + M22*J32 + M23*J33;
+            const double R31 = M31*J11 + M32*J12 + M33*J13;
+            const double R32 = M31*J21 + M32*J22 + M33*J23;
+            const double R33 = M31*J31 + M32*J32 + M33*J33;
+
+            // Now set y to detJ J^{-1} R = adj(J) R
+            y(i11,q,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+            y(i12,q,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
+            y(i13,q,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+            y(i21,q,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
+            y(i22,q,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
+            y(i23,q,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
+            y(i31,q,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+            y(i32,q,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+            y(i33,q,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
+         }
+         else if (coeffDim == 3)  // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double D3 = coeff(2, q, e);
+            // detJ J^{-1} DJ = adj(J) DJ
+            y(i11,q,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
+            y(i12,q,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
+            y(i13,q,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
+            y(i21,q,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
+            y(i22,q,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
+            y(i23,q,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
+            y(i31,q,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
+            y(i32,q,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
+            y(i33,q,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
+         }
+      }
+   });
+}
+
+// PA H(curl) x H(div) mass assemble 2D kernel, with factor
+// dF^{-1} C dF for a vector or matrix coefficient C.
+// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
+void PAHcurlHdivSetup2D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const bool transpose,
+                        const Array<double> &_w,
+                        const Vector &j,
+                        Vector &_coeff,
+                        Vector &op)
+{
+   const int NQ = Q1D*Q1D;
+   const bool symmetric = (coeffDim != 4);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 4, NQ, NE);
+
+   const int i11 = 0;
+   const int i12 = transpose ? 2 : 1;
+   const int i21 = transpose ? 1 : 2;
+   const int i22 = 3;
+
+   MFEM_FORALL(e, NE,
+   {
+      for (int q = 0; q < NQ; ++q)
+      {
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double w_detJ = W[q] / (J11*J22) - (J21*J12);
+
+         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
+         {
+            // First compute entries of R = MJ
+            const double M11 = coeff(i11, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(2, q, e);
+
+            const double R11 = M11*J11 + M12*J21;
+            const double R12 = M11*J12 + M12*J22;
+            const double R21 = M21*J11 + M22*J21;
+            const double R22 = M21*J12 + M22*J22;
+
+            // Now set y to J^{-1} R
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+         }
+         else if (coeffDim == 2) // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double R11 = D1*J11;
+            const double R12 = D1*J12;
+            const double R21 = D2*J21;
+            const double R22 = D2*J22;
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+         }
+      }
+   });
+}
+
+// Mass operator for H(curl) and H(div) functions, using Piola transformations
+// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
+void PAHcurlHdivMassApply3D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const Array<double> &_Bo,
+                            const Array<double> &_Bc,
+                            const Array<double> &_Bot,
+                            const Array<double> &_Bct,
+                            const Vector &_op,
+                            const Vector &_x,
+                            Vector &_y)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+   constexpr static int VDIM = 3;
+
+   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
+   auto Bot = Reshape(_Bot.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(_Bct.Read(), D1Dtest, Q1D);
+   auto op = Reshape(_op.Read(), scalarCoeff ? 1 : 9, Q1D, Q1D, Q1D, NE);
+   auto x = Reshape(_x.Read(), 3*(D1D-1)*D1D*(trialHcurl ? D1D : D1D-1), NE);
+   auto y = Reshape(_y.ReadWrite(), 3*(D1Dtest-1)*D1Dtest*
+                    (trialHcurl ? D1Dtest-1 : D1Dtest), NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  mass[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z trial components
+      {
+         const int D1Dz = trialHcurl ? ((c == 2) ? D1D - 1 : D1D) :
+                          ((c == 2) ? D1D : D1D - 1);
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
+                          ((c == 1) ? D1D : D1D - 1);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
+                          ((c == 0) ? D1D : D1D - 1);
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double massXY[MAX_Q1D][MAX_Q1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massXY[qy][qx] = 0.0;
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massX[MAX_Q1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] = 0.0;
+               }
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double t = x(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
+                                       ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
+                                    ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx = massX[qx];
+                     massXY[qy][qx] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = trialHcurl ? ((c == 2) ? Bo(qz,dz) : Bc(qz,dz)) :
+                                 ((c == 2) ? Bc(qz,dz) : Bo(qz,dz));
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(0,qx,qy,qz,e);
+               const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,qz,e);
+               const double O13 = scalarCoeff ? 0.0 : op(2,qx,qy,qz,e);
+               const double O21 = scalarCoeff ? 0.0 : op(3,qx,qy,qz,e);
+               const double O22 = scalarCoeff ? O11 : op(4,qx,qy,qz,e);
+               const double O23 = scalarCoeff ? 0.0 : op(5,qx,qy,qz,e);
+               const double O31 = scalarCoeff ? 0.0 : op(6,qx,qy,qz,e);
+               const double O32 = scalarCoeff ? 0.0 : op(7,qx,qy,qz,e);
+               const double O33 = scalarCoeff ? O11 : op(8,qx,qy,qz,e);
+               const double massX = mass[qz][qy][qx][0];
+               const double massY = mass[qz][qy][qx][1];
+               const double massZ = mass[qz][qy][qx][2];
+               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
+               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
+            }
+         }
+      }
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double massXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z test components
+         {
+            const int D1Dz = trialHcurl ? ((c == 2) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 2) ? D1Dtest - 1 : D1Dtest);
+            const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 1) ? D1Dtest - 1 : D1Dtest);
+            const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 0) ? D1Dtest - 1 : D1Dtest);
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massXY[dy][dx] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massX[HDIV_MAX_D1D];
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] = 0.0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massX[dx] += mass[qz][qy][qx][c] * (trialHcurl ?
+                                                         ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
+                                                         ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
+                                    ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massXY[dy][dx] += massX[dx] * wy;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = trialHcurl ? ((c == 2) ? Bct(dz,qz) : Bot(dz,qz)) :
+                                 ((c == 2) ? Bot(dz,qz) : Bct(dz,qz));
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) +=
+                        massXY[dy][dx] * wz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         }  // loop c
+      }  // loop qz
+   }); // end of element loop
+}
+
+// Mass operator for H(curl) and H(div) functions, using Piola transformations
+// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
+void PAHcurlHdivMassApply2D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const Array<double> &_Bo,
+                            const Array<double> &_Bc,
+                            const Array<double> &_Bot,
+                            const Array<double> &_Bct,
+                            const Vector &_op,
+                            const Vector &_x,
+                            Vector &_y)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+   constexpr static int VDIM = 2;
+
+   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
+   auto Bot = Reshape(_Bot.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(_Bct.Read(), D1Dtest, Q1D);
+   auto op = Reshape(_op.Read(), scalarCoeff ? 1 : 4, Q1D, Q1D, NE);
+   auto x = Reshape(_x.Read(), 2*(D1D-1)*D1D, NE);
+   auto y = Reshape(_y.ReadWrite(), 2*(D1Dtest-1)*D1Dtest, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      double mass[MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               mass[qy][qx][c] = 0.0;
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y trial components
+      {
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
+                          ((c == 1) ? D1D : D1D - 1);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
+                          ((c == 0) ? D1D : D1D - 1);
+
+         for (int dy = 0; dy < D1Dy; ++dy)
+         {
+            double massX[MAX_Q1D];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               massX[qx] = 0.0;
+            }
+
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               const double t = x(dx + (dy * D1Dx) + osc, e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
+                                    ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
+               }
+            }
+
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
+                                 ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  mass[qy][qx][c] += massX[qx] * wy;
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double O11 = op(0,qx,qy,e);
+            const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,e);
+            const double O21 = scalarCoeff ? 0.0 : op(2,qx,qy,e);
+            const double O22 = scalarCoeff ? O11 : op(3,qx,qy,e);
+            const double massX = mass[qy][qx][0];
+            const double massY = mass[qy][qx][1];
+            mass[qy][qx][0] = (O11*massX)+(O12*massY);
+            mass[qy][qx][1] = (O21*massX)+(O22*massY);
+         }
+      }
+
+      osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y test components
+      {
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
+                          ((c == 1) ? D1Dtest - 1 : D1Dtest);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
+                          ((c == 0) ? D1Dtest - 1 : D1Dtest);
+
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double massX[HDIV_MAX_D1D];
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               massX[dx] = 0.0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] += mass[qy][qx][c] * (trialHcurl ?
+                                                  ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
+                                                  ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
+               }
+            }
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
+                                 ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  y(dx + (dy * D1Dx) + osc, e) += massX[dx] * wy;
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy;
+      }  // loop c
+   }); // end of element loop
+}
+
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   // Assumes tensor-product elements
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement *fel = fes.GetFE(0);
+   AssemblePA(fes, fes);
+}
 
-   const VectorTensorFiniteElement *el =
-      dynamic_cast<const VectorTensorFiniteElement*>(fel);
-   MFEM_VERIFY(el != NULL, "Only VectorTensorFiniteElement is supported!");
+void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
+                                        const FiniteElementSpace &test_fes)
+{
+   // Assumes tensor-product elements
+   Mesh *mesh = trial_fes.GetMesh();
+
+   const FiniteElement *trial_fel = trial_fes.GetFE(0);
+   const VectorTensorFiniteElement *trial_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
+   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const FiniteElement *test_fel = test_fes.GetFE(0);
+   const VectorTensorFiniteElement *test_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
+   MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
 
    const IntegrationRule *ir
-      = IntRule ? IntRule : &MassIntegrator::GetRule(*el, *el,
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
                                                      *mesh->GetElementTransformation(0));
-   const int dims = el->GetDim();
+   const int dims = trial_el->GetDim();
    MFEM_VERIFY(dims == 2 || dims == 3, "");
 
    const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
@@ -161,53 +721,160 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
-   ne = fes.GetNE();
+   ne = trial_fes.GetNE();
+   MFEM_VERIFY(ne == test_fes.GetNE(),
+               "Different meshes for test and trial spaces");
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   mapsO = &el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
    dofs1D = mapsC->ndof;
    quad1D = mapsC->nqpt;
 
+   mapsCtest = &test_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsOtest = &test_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1Dtest = mapsCtest->ndof;
+
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
+   trial_fetype = trial_el->GetDerivType();
+   test_fetype = test_el->GetDerivType();
 
-   Vector coeff(ne * nq);
+   const int MQsymmDim = MQ ? (MQ->GetWidth() * (MQ->GetWidth() + 1)) / 2 : 0;
+   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
+   const int MQdim = MQ ? (MQ->IsSymmetric() ? MQsymmDim : MQfullDim) : 0;
+   const int coeffDim = MQ ? MQdim : (VQ ? VQ->GetVDim() : 1);
+
+   symmetric = MQ ? MQ->IsSymmetric() : true;
+
+   if ((trial_fetype == mfem::FiniteElement::CURL &&
+        test_fetype == mfem::FiniteElement::DIV) ||
+       (trial_fetype == mfem::FiniteElement::DIV &&
+        test_fetype == mfem::FiniteElement::CURL))
+      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
+                      Device::GetMemoryType());
+   else
+      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
+                      Device::GetMemoryType());
+
+   Vector coeff(coeffDim * ne * nq);
    coeff = 1.0;
-   if (Q)
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || VQ || MQ)
    {
+      Vector D(VQ ? coeffDim : 0);
+      DenseMatrix M;
+      Vector Msymm;
+      if (MQ)
+      {
+         if (symmetric)
+         {
+            Msymm.SetSize(MQsymmDim);
+         }
+         else
+         {
+            M.SetSize(dim);
+         }
+      }
+
+      if (VQ)
+      {
+         MFEM_VERIFY(coeffDim == dim, "");
+      }
+      if (MQ)
+      {
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+      }
+
       for (int e=0; e<ne; ++e)
       {
          ElementTransformation *tr = mesh->GetElementTransformation(e);
          for (int p=0; p<nq; ++p)
          {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+            if (MQ)
+            {
+               if (MQ->IsSymmetric())
+               {
+                  MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<MQsymmDim; ++i)
+                  {
+                     coeffh(i, p, e) = Msymm[i];
+                  }
+               }
+               else
+               {
+                  MQ->Eval(M, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<dim; ++i)
+                     for (int j=0; j<dim; ++j)
+                     {
+                        coeffh(j+(i*dim), p, e) = M(i,j);
+                     }
+               }
+            }
+            else if (VQ)
+            {
+               VQ->Eval(D, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = D[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
          }
       }
    }
 
-   fetype = el->GetDerivType();
-
-   if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
+   if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype
+       && dim == 3)
    {
-      PAHcurlSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
-   else if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
+   else if (trial_fetype == mfem::FiniteElement::CURL
+            && test_fetype == trial_fetype && dim == 2)
    {
-      PAHcurlSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
-   else if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
+   else if (trial_fetype == mfem::FiniteElement::DIV
+            && test_fetype == trial_fetype && dim == 3)
    {
       PAHdivSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
-   else if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 2)
+   else if (trial_fetype == mfem::FiniteElement::DIV
+            && test_fetype == trial_fetype && dim == 2)
    {
       PAHdivSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
+   else if (((trial_fetype == mfem::FiniteElement::CURL &&
+              test_fetype == mfem::FiniteElement::DIV) ||
+             (trial_fetype == mfem::FiniteElement::DIV &&
+              test_fetype == mfem::FiniteElement::CURL)) &&
+            test_fel->GetOrder() == trial_fel->GetOrder())
+   {
+      if (coeffDim == 1)
+      {
+         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+      }
+      else
+      {
+         const bool tr = (trial_fetype == mfem::FiniteElement::DIV &&
+                          test_fetype == mfem::FiniteElement::CURL);
+         if (dim == 3)
+            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+                               geom->J, coeff, pa_data);
+         else
+            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+                               geom->J, coeff, pa_data);
+      }
+   }
    else
    {
       MFEM_ABORT("Unknown kernel.");
@@ -218,12 +885,13 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 {
    if (dim == 3)
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne,
+         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
                                        mapsO->B, mapsC->B, pa_data, diag);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
@@ -235,12 +903,13 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
    }
    else
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         PAHcurlMassAssembleDiagonal2D(dofs1D, quad1D, ne,
+         PAHcurlMassAssembleDiagonal2D(dofs1D, quad1D, ne, symmetric,
                                        mapsO->B, mapsC->B, pa_data, diag);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
@@ -256,16 +925,33 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
    if (dim == 3)
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         PAHcurlMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                            mapsC->Bt, pa_data, x, y);
       }
+      else if (trial_fetype == mfem::FiniteElement::CURL &&
+               test_fetype == mfem::FiniteElement::DIV)
+      {
+         const bool scalarCoeff = !(VQ || MQ);
+         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                true, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                mapsCtest->Bt, pa_data, x, y);
+      }
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == mfem::FiniteElement::CURL)
+      {
+         const bool scalarCoeff = !(VQ || MQ);
+         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                mapsCtest->Bt, pa_data, x, y);
+      }
       else
       {
          MFEM_ABORT("Unknown kernel.");
@@ -273,16 +959,28 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      if (fetype == mfem::FiniteElement::CURL)
+      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
-         PAHcurlMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                            mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
       }
-      else if (fetype == mfem::FiniteElement::DIV)
+      else if (trial_fetype == mfem::FiniteElement::DIV &&
+               test_fetype == trial_fetype)
       {
          PAHdivMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                            mapsC->Bt, pa_data, x, y);
       }
+      else if ((trial_fetype == mfem::FiniteElement::CURL &&
+                test_fetype == mfem::FiniteElement::DIV) ||
+               (trial_fetype == mfem::FiniteElement::DIV &&
+                test_fetype == mfem::FiniteElement::CURL))
+      {
+         const bool scalarCoeff = !(VQ || MQ);
+         const bool trialHcurl = (trial_fetype == mfem::FiniteElement::CURL);
+         PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                trialHcurl, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                mapsCtest->Bt, pa_data, x, y);
+      }
       else
       {
          MFEM_ABORT("Unknown kernel.");
@@ -348,12 +1046,12 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
    // Use the same setup functions as VectorFEMassIntegrator.
    if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
    {
-      PAHcurlSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
    else if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
    {
-      PAHcurlSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup2D(quad1D, 1, ne, ir->GetWeights(), geom->J,
                      coeff, pa_data);
    }
    else

fem/coefficient.cpp

@@ -209,18 +209,24 @@ void GradientGridFunctionCoefficient::Eval(
    GridFunc->GetGradients(T, ir, M);
 }
 
-CurlGridFunctionCoefficient::CurlGridFunctionCoefficient (
+CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
    const GridFunction *gf)
-   : VectorCoefficient ((gf) ?
-                        gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0)
+   : VectorCoefficient(0)
 {
-   GridFunc = gf;
+   SetGridFunction(gf);
 }
 
 void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 {
-   GridFunc = gf; vdim = (gf) ?
-                         gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0;
+   if (gf)
+   {
+      int sdim = gf -> FESpace() -> GetMesh() -> SpaceDimension();
+      MFEM_VERIFY(sdim == 2 || sdim == 3,
+                  "CurlGridFunctionCoefficient "
+                  "only defind for spaces of dimension 2 or 3.");
+   }
+   GridFunc = gf;
+   vdim = (gf) ? (2 * gf -> FESpace() -> GetMesh() -> SpaceDimension() - 3) : 0;
 }
 
 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
@@ -313,6 +319,31 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
    }
 }
 
+void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
+                                              ElementTransformation &T,
+                                              const IntegrationPoint &ip)
+{
+   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
+               "MatrixFunctionCoefficient is not symmetric");
+
+   double x[3];
+   Vector transip(x, 3);
+
+   T.Transform(ip, transip);
+
+   K.SetSize((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
+
+   if (SymmFunction)
+   {
+      (*SymmFunction)(transip, K);
+   }
+
+   if (Q)
+   {
+      K *= Q->Eval(T, ip, GetTime());
+   }
+}
+
 MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
    : MatrixCoefficient (dim)
 {
@@ -416,13 +447,43 @@ double DeterminantCoefficient::Eval(ElementTransformation &T,
    return ma.Det();
 }
 
-VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &A,
-                                           VectorCoefficient &B,
-                                           double _alpha, double _beta)
-   : VectorCoefficient(A.GetVDim()), a(&A), b(&B), alpha(_alpha), beta(_beta),
-     va(A.GetVDim())
+VectorSumCoefficient::VectorSumCoefficient(int dim)
+   : VectorCoefficient(dim),
+     ACoef(NULL), BCoef(NULL),
+     A(dim), B(dim),
+     alphaCoef(NULL), betaCoef(NULL),
+     alpha(1.0), beta(1.0)
 {
-   MFEM_ASSERT(A.GetVDim() == B.GetVDim(),
+   A = 0.0; B = 0.0;
+}
+
+VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &_A,
+                                           VectorCoefficient &_B,
+                                           double _alpha, double _beta)
+   : VectorCoefficient(_A.GetVDim()),
+     ACoef(&_A), BCoef(&_B),
+     A(_A.GetVDim()), B(_A.GetVDim()),
+     alphaCoef(NULL), betaCoef(NULL),
+     alpha(_alpha), beta(_beta)
+{
+   MFEM_ASSERT(_A.GetVDim() == _B.GetVDim(),
+               "VectorSumCoefficient:  "
+               "Arguments must have the same dimension.");
+}
+
+VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &_A,
+                                           VectorCoefficient &_B,
+                                           Coefficient &_alpha,
+                                           Coefficient &_beta)
+   : VectorCoefficient(_A.GetVDim()),
+     ACoef(&_A), BCoef(&_B),
+     A(_A.GetVDim()),
+     B(_A.GetVDim()),
+     alphaCoef(&_alpha),
+     betaCoef(&_beta),
+     alpha(0.0), beta(0.0)
+{
+   MFEM_ASSERT(_A.GetVDim() == _B.GetVDim(),
                "VectorSumCoefficient:  "
                "Arguments must have the same dimension.");
 }
@@ -430,26 +491,47 @@ VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &A,
 void VectorSumCoefficient::Eval(Vector &V, ElementTransformation &T,
                                 const IntegrationPoint &ip)
 {
-   b->Eval(V, T, ip);
-   if ( beta != 1.0 ) { V *= beta; }
-   a->Eval(va, T, ip);
-   V.Add(alpha, va);
+   V.SetSize(A.Size());
+   if (    ACoef) { ACoef->Eval(A, T, ip); }
+   if (    BCoef) { BCoef->Eval(B, T, ip); }
+   if (alphaCoef) { alpha = alphaCoef->Eval(T, ip); }
+   if ( betaCoef) { beta  = betaCoef->Eval(T, ip); }
+   add(alpha, A, beta, B, V);
 }
 
+ScalarVectorProductCoefficient::ScalarVectorProductCoefficient(
+   double A,
+   VectorCoefficient &B)
+   : VectorCoefficient(B.GetVDim()), aConst(A), a(NULL), b(&B)
+{}
+
 ScalarVectorProductCoefficient::ScalarVectorProductCoefficient(
    Coefficient &A,
    VectorCoefficient &B)
-   : VectorCoefficient(B.GetVDim()), a(&A), b(&B)
+   : VectorCoefficient(B.GetVDim()), aConst(0.0), a(&A), b(&B)
 {}
 
 void ScalarVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
                                           const IntegrationPoint &ip)
 {
-   double sa = a->Eval(T, ip);
+   double sa = (a == NULL) ? aConst : a->Eval(T, ip);
    b->Eval(V, T, ip);
    V *= sa;
 }
 
+NormalizedVectorCoefficient::NormalizedVectorCoefficient(VectorCoefficient &A,
+                                                         double _tol)
+   : VectorCoefficient(A.GetVDim()), a(&A), tol(_tol)
+{}
+
+void NormalizedVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
+                                       const IntegrationPoint &ip)
+{
+   a->Eval(V, T, ip);
+   double nv = V.Norml2();
+   V *= (nv > tol) ? (1.0/nv) : 0.0;
+}
+
 VectorCrossProductCoefficient::VectorCrossProductCoefficient(
    VectorCoefficient &A,
    VectorCoefficient &B)
@@ -471,17 +553,18 @@ void VectorCrossProductCoefficient::Eval(Vector &V, ElementTransformation &T,
    V[2] = va[0] * vb[1] - va[1] * vb[0];
 }
 
-MatVecCoefficient::MatVecCoefficient(MatrixCoefficient &A,
-                                     VectorCoefficient &B)
+MatrixVectorProductCoefficient::MatrixVectorProductCoefficient(
+   MatrixCoefficient &A, VectorCoefficient &B)
    : VectorCoefficient(A.GetHeight()), a(&A), b(&B),
      ma(A.GetHeight(), A.GetWidth()), vb(B.GetVDim())
 {
    MFEM_ASSERT(A.GetWidth() == B.GetVDim(),
-               "MatVecCoefficient:  Arguments have incompatible dimensions.");
+               "MatrixVectorProductCoefficient:  "
+               "Arguments have incompatible dimensions.");
 }
 
-void MatVecCoefficient::Eval(Vector &V, ElementTransformation &T,
-                             const IntegrationPoint &ip)
+void MatrixVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
+                                          const IntegrationPoint &ip)
 {
    a->Eval(ma, T, ip);
    b->Eval(vb, T, ip);
@@ -517,17 +600,23 @@ void MatrixSumCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
    M.Add(alpha, ma);
 }
 
+ScalarMatrixProductCoefficient::ScalarMatrixProductCoefficient(
+   double A,
+   MatrixCoefficient &B)
+   : MatrixCoefficient(B.GetHeight(), B.GetWidth()), aConst(A), a(NULL), b(&B)
+{}
+
 ScalarMatrixProductCoefficient::ScalarMatrixProductCoefficient(
    Coefficient &A,
    MatrixCoefficient &B)
-   : MatrixCoefficient(B.GetHeight(), B.GetWidth()), a(&A), b(&B)
+   : MatrixCoefficient(B.GetHeight(), B.GetWidth()), aConst(0.0), a(&A), b(&B)
 {}
 
 void ScalarMatrixProductCoefficient::Eval(DenseMatrix &M,
                                           ElementTransformation &T,
                                           const IntegrationPoint &ip)
 {
-   double sa = a->Eval(T, ip);
+   double sa = (a == NULL) ? aConst : a->Eval(T, ip);
    b->Eval(M, T, ip);
    M *= sa;
 }
@@ -581,6 +670,30 @@ void OuterProductCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
    }
 }
 
+CrossCrossCoefficient::CrossCrossCoefficient(Coefficient &A,
+                                             VectorCoefficient &K)
+   : MatrixCoefficient(K.GetVDim(), K.GetVDim()), aConst(0.0), a(&A), k(&K),
+     vk(K.GetVDim())
+{}
+
+void CrossCrossCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
+                                 const IntegrationPoint &ip)
+{
+   k->Eval(vk, T, ip);
+   M.SetSize(vk.Size(), vk.Size());
+   M = 0.0;
+   double k2 = vk*vk;
+   for (int i=0; i<vk.Size(); i++)
+   {
+      M(i, i) = k2;
+      for (int j=0; j<vk.Size(); j++)
+      {
+         M(i, j) -= vk[i] * vk[j];
+      }
+   }
+   M *= ((a == NULL ) ? aConst : a->Eval(T, ip) );
+}
+
 double LpNormLoop(double p, Coefficient &coeff, Mesh &mesh,
                   const IntegrationRule *irs[])
 {

fem/complex_fem.hpp

@@ -99,8 +99,8 @@ public:
                      ComplexOperator::Convention
                      convention = ComplexOperator::HERMITIAN);
 
-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
-       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
+       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
 
        The pointer @a fes is not owned by the newly constructed object.
 
@@ -195,8 +195,8 @@ private:
    BilinearForm *blfr;
    BilinearForm *blfi;
 
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
+   /* These methods check if the real/imag parts of the sesquilinear form are
+      not empty */
    bool RealInteg();
    bool ImagInteg();
 
@@ -204,7 +204,7 @@ public:
    SesquilinearForm(FiniteElementSpace *fes,
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
        the same integrators as the BilinearForms @a bfr and @a bfi .
 
        The pointer @a fes is not owned by the newly constructed object.
@@ -219,6 +219,21 @@ public:
    void SetConvention(const ComplexOperator::Convention &
                       convention) { conv = convention; }
 
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::FULL  (default)
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level)
+   {
+      blfr->SetAssemblyLevel(assembly_level);
+      blfi->SetAssemblyLevel(assembly_level);
+   }
+
    BilinearForm & real() { return *blfr; }
    BilinearForm & imag() { return *blfi; }
    const BilinearForm & real() const { return *blfr; }
@@ -309,7 +324,7 @@ protected:
 public:
 
    /* @brief Construct a ParComplexGridFunction associated with the
-      ParFiniteElementSpace @a *f. */
+      ParFiniteElementSpace @a *pf. */
    ParComplexGridFunction(ParFiniteElementSpace *pf);
 
    void Update();
@@ -401,8 +416,8 @@ public:
                         convention = ComplexOperator::HERMITIAN);
 
    /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plfr (real) and @a plfi
-       (imag) .
+       using the same integrators as the LinearForms @a plf_r (real) and
+       @a plf_i (imag).
 
       The pointer @a fes is not owned by the newly constructed object.
 
@@ -478,7 +493,7 @@ public:
 /** Class for a parallel sesquilinear form
 
     A sesquilinear form is a generalization of a bilinear form to complex-valued
-    fields. Sesquilinear forms are linear in the second argument but but the
+    fields. Sesquilinear forms are linear in the second argument but the
     first argument involves a complex conjugate in the sense that:
 
                 a(alpha u, beta v) = conj(alpha) beta a(u, v)
@@ -524,6 +539,21 @@ public:
    void SetConvention(const ComplexOperator::Convention &
                       convention) { conv = convention; }
 
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::FULL  (default)
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level)
+   {
+      pblfr->SetAssemblyLevel(assembly_level);
+      pblfi->SetAssemblyLevel(assembly_level);
+   }
+
    ParBilinearForm & real() { return *pblfr; }
    ParBilinearForm & imag() { return *pblfi; }
    const ParBilinearForm & real() const { return *pblfr; }

fem/datacollection.cpp

@@ -415,9 +415,6 @@ void VisItDataCollection::SetMesh(MPI_Comm comm, Mesh *new_mesh)
 void VisItDataCollection::RegisterField(const std::string& name,
                                         GridFunction *gf)
 {
-   DataCollection::RegisterField(name, gf);
-   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim());
-
    int LOD = 1;
    if (gf->FESpace()->GetNURBSext())
    {
@@ -431,6 +428,27 @@ void VisItDataCollection::RegisterField(const std::string& name,
       }
    }
 
+   DataCollection::RegisterField(name, gf);
+   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim(), LOD);
+   visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
+}
+
+void VisItDataCollection::RegisterQField(const std::string& name,
+                                         QuadratureFunction *qf)
+{
+   int LOD = -1;
+   Mesh *mesh = qf->GetSpace()->GetMesh();
+   for (int e=0; e<qf->GetSpace()->GetNE(); e++)
+   {
+      int locLOD = GlobGeometryRefiner.GetRefinementLevelFromElems(
+                      mesh->GetElementBaseGeometry(e),
+                      qf->GetElementIntRule(e).GetNPoints());
+
+      LOD = std::max(LOD,locLOD);
+   }
+
+   DataCollection::RegisterQField(name, qf);
+   field_info_map[name] = VisItFieldInfo("elements", 1, LOD);
    visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
 }
 
@@ -598,14 +616,28 @@ void VisItDataCollection::LoadFields()
       // TODO: 1) load parallel GridFunction on one processor
       if (serial)
       {
-         field_map.Register(it->first, new GridFunction(mesh, file), own_data);
+         if ((it->second).association == "nodes")
+         {
+            field_map.Register(it->first, new GridFunction(mesh, file), own_data);
+         }
+         else if ((it->second).association == "elements")
+         {
+            q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
+         }
       }
       else
       {
 #ifdef MFEM_USE_MPI
-         field_map.Register(
-            it->first,
-            new ParGridFunction(dynamic_cast<ParMesh*>(mesh), file), own_data);
+         if ((it->second).association == "nodes")
+         {
+            field_map.Register(
+               it->first,
+               new ParGridFunction(dynamic_cast<ParMesh*>(mesh), file), own_data);
+         }
+         else if ((it->second).association == "elements")
+         {
+            q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
+         }
 #else
          error = READ_ERROR;
          MFEM_WARNING("Reading parallel format in serial is not supported");
@@ -640,7 +672,7 @@ std::string VisItDataCollection::GetVisItRootString()
    {
       ftags["assoc"] = picojson::value((it->second).association);
       ftags["comps"] = picojson::value(to_string((it->second).num_components));
-      ftags["lod"] = picojson::value(to_string(visit_levels_of_detail));
+      ftags["lod"] = picojson::value(to_string((it->second).lod));
       field["path"] = picojson::value(path_str + it->first + file_ext_format);
       field["tags"] = picojson::value(ftags);
       fields[it->first] = picojson::value(field);

fem/datacollection.hpp

@@ -391,9 +391,10 @@ class VisItFieldInfo
 public:
    std::string association;
    int num_components;
-   VisItFieldInfo() { association = ""; num_components = 0; }
-   VisItFieldInfo(std::string _association, int _num_components)
-   { association = _association; num_components = _num_components; }
+   int lod;
+   VisItFieldInfo() { association = ""; num_components = 0; lod = 1;}
+   VisItFieldInfo(std::string _association, int _num_components, int _lod = 1)
+   { association = _association; num_components = _num_components; lod =_lod;}
 };
 
 /// Data collection with VisIt I/O routines
@@ -445,6 +446,12 @@ public:
    /// Add a grid function to the collection and update the root file
    virtual void RegisterField(const std::string& field_name, GridFunction *gf);
 
+   /// Add a quadrature function to the collection and update the root file.
+   /** Visualization of quadrature function is not supported in VisIt(3.12).
+       A patch has been sent to VisIt developers in June 2020. */
+   virtual void RegisterQField(const std::string& q_field_name,
+                               QuadratureFunction *qf);
+
    /// Set VisIt parameter: default levels of detail for the MultiresControl
    void SetLevelsOfDetail(int levels_of_detail);
 

fem/eltrans.cpp

@@ -552,26 +552,32 @@ void IntegrationPointTransformation::Transform (const IntegrationRule &ir1,
    }
 }
 
-void FaceElementTransformations::SetIntPoint(const IntegrationPoint *ip)
+void FaceElementTransformations::SetIntPoint(const IntegrationPoint *face_ip)
 {
-   IsoparametricTransformation::SetIntPoint(ip);
+   IsoparametricTransformation::SetIntPoint(face_ip);
 
-   if (Elem1)
+   if (mask & 4)
    {
-      Loc1.Transform(*ip, eip1);
-      Elem1->SetIntPoint(&eip1);
+      Loc1.Transform(*face_ip, eip1);
+      if (Elem1)
+      {
+         Elem1->SetIntPoint(&eip1);
+      }
    }
-   if (Elem2)
+   if (mask & 8)
    {
-      Loc2.Transform(*ip, eip2);
-      Elem2->SetIntPoint(&eip2);
+      Loc2.Transform(*face_ip, eip2);
+      if (Elem2)
+      {
+         Elem2->SetIntPoint(&eip2);
+      }
    }
 }
 
 ElementTransformation &
 FaceElementTransformations::GetElement1Transformation()
 {
-   MFEM_VERIFY(mask & 1 && Elem1 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_ELEM1 && Elem1 != NULL, "The ElementTransformation "
                "for the element has not been configured for side 1.");
    return *Elem1;
 }
@@ -579,7 +585,7 @@ FaceElementTransformations::GetElement1Transformation()
 ElementTransformation &
 FaceElementTransformations::GetElement2Transformation()
 {
-   MFEM_VERIFY(mask & 2 && Elem2 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_ELEM2 && Elem2 != NULL, "The ElementTransformation "
                "for the element has not been configured for side 2.");
    return *Elem2;
 }
@@ -587,7 +593,7 @@ FaceElementTransformations::GetElement2Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint1Transformation()
 {
-   MFEM_VERIFY(mask & 4, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & HAVE_LOC1, "The IntegrationPointTransformation "
                "for the element has not been configured for side 1.");
    return Loc1;
 }
@@ -595,7 +601,7 @@ FaceElementTransformations::GetIntPoint1Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint2Transformation()
 {
-   MFEM_VERIFY(mask & 8, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & HAVE_LOC2, "The IntegrationPointTransformation "
                "for the element has not been configured for side 2.");
    return Loc2;
 }
@@ -603,7 +609,7 @@ FaceElementTransformations::GetIntPoint2Transformation()
 void FaceElementTransformations::Transform(const IntegrationPoint &ip,
                                            Vector &trans)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(ip, trans);
 }
@@ -611,7 +617,7 @@ void FaceElementTransformations::Transform(const IntegrationPoint &ip,
 void FaceElementTransformations::Transform(const IntegrationRule &ir,
                                            DenseMatrix &tr)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(ir, tr);
 }
@@ -619,9 +625,82 @@ void FaceElementTransformations::Transform(const IntegrationRule &ir,
 void FaceElementTransformations::Transform(const DenseMatrix &matrix,
                                            DenseMatrix &result)
 {
-   MFEM_VERIFY(mask & 16, "The ElementTransformation "
+   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
                "for the face has not been configured.");
    IsoparametricTransformation::Transform(matrix, result);
 }
 
+double FaceElementTransformations::CheckConsistency(int print_level,
+                                                    std::ostream &out)
+{
+   // Check that the face vertices are mapped to the same physical location
+   // when using the following three transformations:
+   // - the face transformation, *this
+   // - Loc1 + Elem1
+   // - Loc2 + Elem2, if present.
+
+   const bool have_face = (mask & 16);
+   const bool have_el1 = (mask & 1) && (mask & 4);
+   const bool have_el2 = (mask & 2) && (mask & 8) && (Elem2No >= 0);
+   if (int(have_face) + int(have_el1) + int(have_el2) < 2)
+   {
+      // need at least two different transformations to perform a check
+      return 0.0;
+   }
+
+   const IntegrationRule &v_ir = *Geometries.GetVertices(GetGeometryType());
+
+   double max_dist = 0.0;
+   Vector dist(v_ir.GetNPoints());
+   DenseMatrix coords_base, coords_el;
+   IntegrationRule v_eir(v_ir.GetNPoints());
+   if (have_face)
+   {
+      Transform(v_ir, coords_base);
+      if (print_level > 0)
+      {
+         out << "\nface vertex coordinates (from face transform):\n"
+             << "----------------------------------------------\n";
+         coords_base.PrintT(out, coords_base.Height());
+      }
+   }
+   if (have_el1)
+   {
+      Loc1.Transform(v_ir, v_eir);
+      Elem1->Transform(v_eir, coords_el);
+      if (print_level > 0)
+      {
+         out << "\nface vertex coordinates (from element 1 transform):\n"
+             << "---------------------------------------------------\n";
+         coords_el.PrintT(out, coords_el.Height());
+      }
+      if (have_face)
+      {
+         coords_el -= coords_base;
+         coords_el.Norm2(dist);
+         max_dist = std::max(max_dist, dist.Normlinf());
+      }
+      else
+      {
+         coords_base = coords_el;
+      }
+   }
+   if (have_el2)
+   {
+      Loc2.Transform(v_ir, v_eir);
+      Elem2->Transform(v_eir, coords_el);
+      if (print_level > 0)
+      {
+         out << "\nface vertex coordinates (from element 2 transform):\n"
+             << "---------------------------------------------------\n";
+         coords_el.PrintT(out, coords_el.Height());
+      }
+      coords_el -= coords_base;
+      coords_el.Norm2(dist);
+      max_dist = std::max(max_dist, dist.Normlinf());
+   }
+
+   return max_dist;
+}
+
 }

fem/eltrans.hpp

@@ -38,9 +38,12 @@ protected:
    };
    Geometry::Type geom;
 
-   // Evaluate the Jacobian of the transformation at the IntPoint and store it
-   // in dFdx.
+   /** @brief Evaluate the Jacobian of the transformation at the IntPoint and
+       store it in dFdx. */
    virtual const DenseMatrix &EvalJacobian() = 0;
+
+   /** @brief Evaluate the Hessian of the transformation at the IntPoint and
+       store it in d2Fdx2. */
    virtual const DenseMatrix &EvalHessian() = 0;
 
    double EvalWeight();
@@ -74,14 +77,27 @@ public:
 
    ElementTransformation();
 
+   /** @brief Set the integration point @a ip that weights and Jacobians will
+       be evaluated at. */
    void SetIntPoint(const IntegrationPoint *ip)
    { IntPoint = ip; EvalState = 0; }
+
+   /** @brief Get a const reference to the currently set integration point.  This
+       will return NULL if no integration point is set. */
    const IntegrationPoint &GetIntPoint() { return *IntPoint; }
 
+   /** @brief Transform integration point from reference coordinates to
+       physical coordinates and store them in the vector. */
    virtual void Transform(const IntegrationPoint &, Vector &) = 0;
+
+   /** @brief Transform all the integration points from the integration rule
+       from reference coordinates to physical
+       coordinates and store them as column vectors in the matrix. */
    virtual void Transform(const IntegrationRule &, DenseMatrix &) = 0;
 
-   /// Transform columns of 'matrix', store result in 'result'.
+   /** @brief Transform all the integration points from the column vectors
+       of @a matrix from reference coordinates to physical
+       coordinates and store them as column vectors in @a result. */
    virtual void Transform(const DenseMatrix &matrix, DenseMatrix &result) = 0;
 
    /** @brief Return the Jacobian matrix of the transformation at the currently
@@ -92,27 +108,44 @@ public:
    const DenseMatrix &Jacobian()
    { return (EvalState & JACOBIAN_MASK) ? dFdx : EvalJacobian(); }
 
+
+   /** @brief Return the Hessian matrix of the transformation at the currently
+       set IntegrationPoint, using the method SetIntPoint(). */
    const DenseMatrix &Hessian()
    { return (EvalState & HESSIAN_MASK) ? d2Fdx2 : EvalHessian(); }
 
+   /** @brief Return the weight of the Jacobian matrix of the transformation
+       at the currently set IntegrationPoint.
+       The Weight evaluates to \f$ \sqrt{\lvert J^T J \rvert} \f$. */
    double Weight() { return (EvalState & WEIGHT_MASK) ? Wght : EvalWeight(); }
 
+   /** @brief Return the adjugate of the Jacobian matrix of the transformation
+        at the currently set IntegrationPoint. */
    const DenseMatrix &AdjugateJacobian()
    { return (EvalState & ADJUGATE_MASK) ? adjJ : EvalAdjugateJ(); }
 
+   /** @brief Return the inverse of the Jacobian matrix of the transformation
+        at the currently set IntegrationPoint. */
    const DenseMatrix &InverseJacobian()
    { return (EvalState & INVERSE_MASK) ? invJ : EvalInverseJ(); }
 
+   /// Return the order of the current element we are using for the transformation.
    virtual int Order() const = 0;
+
+   /// Return the order of the elements of the Jacobian of the transformation.
    virtual int OrderJ() const = 0;
+
+   /** @brief Return the order of the determinant of the Jacobian (weight)
+       of the transformation. */
    virtual int OrderW() const = 0;
-   /// Order of adj(J)^t.grad(fi)
+
+   /// Return the order of \f$ adj(J)^T \nabla fi \f$
    virtual int OrderGrad(const FiniteElement *fe) const = 0;
 
    /// Return the Geometry::Type of the reference element.
    Geometry::Type GetGeometryType() const { return geom; }
 
-   /// Return the dimension of the reference element.
+   /// Return the topological dimension of the reference element.
    int GetDimension() const { return Geometry::Dimension[geom]; }
 
    /// Get the dimension of the target (physical) space.
@@ -308,7 +341,7 @@ public:
    virtual int Transform(const Vector &pt, IntegrationPoint &ip);
 };
 
-
+/// A standard isoparametric element transformation
 class IsoparametricTransformation : public ElementTransformation
 {
 private:
@@ -318,26 +351,29 @@ private:
    const FiniteElement *FElem;
    DenseMatrix PointMat; // dim x dof
 
-   // Evaluate the Jacobian of the transformation at the IntPoint and store it
-   // in dFdx.
+   /** @brief Evaluate the Jacobian of the transformation at the IntPoint and
+       store it in dFdx. */
    virtual const DenseMatrix &EvalJacobian();
    // Evaluate the Hessian of the transformation at the IntPoint and store it
    // in d2Fdx2.
    virtual const DenseMatrix &EvalHessian();
 public:
+   /// Set the element that will be used to compute the transformations
    void SetFE(const FiniteElement *FE) { FElem = FE; geom = FE->GetGeomType(); }
+
+   /// Get the current element used to compute the transformations
    const FiniteElement* GetFE() const { return FElem; }
 
    /// @brief Set the underlying point matrix describing the transformation.
    /** The dimensions of the matrix are space-dim x dof. The transformation is
        defined as
+           \f$ x = F( \hat x ) = P \phi( \hat x ) \f$
 
-           x = F(xh) = P . phi(xh),
-
-       where xh (x hat) is the reference point, x is the corresponding physical
-       point, P is the point matrix, and phi(xh) is the column-vector of all
-       basis functions evaluated at xh. The columns of P represent the control
-       points in physical space defining the transformation. */
+       where \f$ \hat x \f$  is the reference point, @a x is the corresponding
+       physical point, @a P is the point matrix, and \f$ \phi( \hat x ) \f$ is
+       the column-vector of all basis functions evaluated at \f$ \hat x \f$ .
+       The columns of @a P represent the control points in physical space
+       defining the transformation. */
    void SetPointMat(const DenseMatrix &pm) { PointMat = pm; }
 
    /// Return the stored point matrix.
@@ -346,19 +382,44 @@ public:
    /// Write access to the stored point matrix. Use with caution.
    DenseMatrix &GetPointMat() { return PointMat; }
 
+   /// Set the FiniteElement Geometry for the reference elements being used.
    void SetIdentityTransformation(Geometry::Type GeomType);
 
+   /** @brief Transform integration point from reference coordinates to
+       physical coordinates and store them in the vector. */
    virtual void Transform(const IntegrationPoint &, Vector &);
+
+   /** @brief Transform all the integration points from the integration rule
+       from reference coordinates to physical
+      coordinates and store them as column vectors in the matrix. */
    virtual void Transform(const IntegrationRule &, DenseMatrix &);
+
+   /** @brief Transform all the integration points from the column vectors
+       of @a matrix from reference coordinates to physical
+       coordinates and store them as column vectors in @a result. */
    virtual void Transform(const DenseMatrix &matrix, DenseMatrix &result);
 
+   /// Return the order of the current element we are using for the transformation.
    virtual int Order() const { return FElem->GetOrder(); }
+
+   /// Return the order of the elements of the Jacobian of the transformation.
    virtual int OrderJ() const;
+
+   /** @brief Return the order of the determinant of the Jacobian (weight)
+       of the transformation. */
    virtual int OrderW() const;
+
+   /// Return the order of \f$ adj(J)^T \nabla fi \f$
    virtual int OrderGrad(const FiniteElement *fe) const;
 
    virtual int GetSpaceDim() const { return PointMat.Height(); }
 
+   /** @brief Transform a point @a pt from physical space to a point @a ip in
+       reference space. */
+   /** Attempt to find the IntegrationPoint that is transformed into the given
+       point in physical space. If the inversion fails a non-zero value is
+       returned. This method is not 100 percent reliable for non-linear
+       transformations. */
    virtual int TransformBack(const Vector & v, IntegrationPoint & ip)
    {
       InverseElementTransformation inv_tr(this);
@@ -378,14 +439,57 @@ public:
    void Transform (const IntegrationRule  &, IntegrationRule  &);
 };
 
+/** @brief A specialized ElementTransformation class representing a face and
+    its two neighboring elements.
+
+    This class can be used as a container for the element transformation data
+    needed for integrating discontinuous fields on element interfaces in a
+    Discontinuous Galerkin (DG) context.
+
+    The secondary purpose of this class is to enable the
+    GridFunction::GetValue function, and various related functions, to properly
+    evaluate fields with limited continuity on boundary elements.
+*/
 class FaceElementTransformations : public IsoparametricTransformation
 {
 private:
+
+   // Bitwise OR of ConfigMasks
    int mask;
 
    IntegrationPoint eip1, eip2;
 
+protected: // interface for Mesh to be able to configure this object.
+
+   friend class Mesh;
+#ifdef MFEM_USE_MPI
+   friend class ParMesh;
+#endif
+
+   /// Set the mask indicating which portions of the object have been setup
+   /** The argument @a m is a bitmask used in
+       Mesh::GetFaceElementTransformations to indicate which portions of the
+       FaceElementTransformations object have been configured.
+
+       mask &  1: Elem1 is configured
+       mask &  2: Elem2 is configured
+       mask &  4: Loc1 is configured
+       mask &  8: Loc2 is configured
+       mask & 16: The Face transformation itself is configured
+   */
+   void SetConfigurationMask(int m) { mask = m; }
+
 public:
+
+   enum ConfigMasks
+   {
+      HAVE_ELEM1 =  1, ///< Element on side 1 is configured
+      HAVE_ELEM2 =  2, ///< Element on side 2 is configured
+      HAVE_LOC1  =  4, ///< Point transformation for side 1 is configured
+      HAVE_LOC2  =  8, ///< Point transformation for side 2 is configured
+      HAVE_FACE  = 16  ///< Face transformation is configured
+   };
+
    int Elem1No, Elem2No;
    Geometry::Type &FaceGeom; ///< @deprecated Use GetGeometryType instead
    ElementTransformation *Elem1, *Elem2;
@@ -404,10 +508,10 @@ public:
    */
    void SetGeometryType(Geometry::Type g) { geom = g; }
 
-   /// Set the mask indicating which portions of the object have been setup
-   /** The argument @a m is a bitmask used in
-       Mesh::GetFaceElementTransformations to indicate which portions of the
-       FaceElement Transformations object have been configured.
+   /** @brief Return the mask defining the configuration state.
+
+       The mask value indicates which portions of FaceElementTransformations
+       object have been configured.
 
        mask &  1: Elem1 is configured
        mask &  2: Elem2 is configured
@@ -415,12 +519,45 @@ public:
        mask &  8: Loc2 is configured
        mask & 16: The Face transformation itself is configured
    */
-   void SetConfigurationMask(int m) { mask = m; }
-   int  GetConfigurationMask() const { return mask; }
+   int GetConfigurationMask() const { return mask; }
 
    /** @brief Set the integration point in the Face and the two neighboring
-       elements, if present. */
-   void SetIntPoint(const IntegrationPoint *ip);
+       elements, if present.
+
+       The point @a face_ip must be in the reference coordinate system of the
+       face.
+   */
+   void SetIntPoint(const IntegrationPoint *face_ip);
+
+   /** @brief Set the integration point in the Face and the two neighboring
+       elements, if present.
+
+       This is a more expressive member function name than SetIntPoint, which
+       in this special case, does the same thing. This function can be used for
+       greater code clarity.
+   */
+   inline void SetAllIntPoints(const IntegrationPoint *face_ip)
+   { FaceElementTransformations::SetIntPoint(face_ip); }
+
+   /** @brief Get a const reference to the integration point in neighboring
+       element 1 corresponding to the currently set integration point on the
+       face.
+
+       This IntegrationPoint object will only contain up-to-date data if
+       SetIntPoint or SetAllIntPoints has been called with the latest
+       integration point for the face and the appropriate point transformation
+       has been configured. */
+   const IntegrationPoint &GetElement1IntPoint() { return eip1; }
+
+   /** @brief Get a const reference to the integration point in neighboring
+       element 2 corresponding to the currently set integration point on the
+       face.
+
+       This IntegrationPoint object will only contain up-to-date data if
+       SetIntPoint or SetAllIntPoints has been called with the latest
+       integration point for the face and the appropriate point transformation
+       has been configured. */
+   const IntegrationPoint &GetElement2IntPoint() { return eip2; }
 
    virtual void Transform(const IntegrationPoint &, Vector &);
    virtual void Transform(const IntegrationRule &, DenseMatrix &);
@@ -430,9 +567,29 @@ public:
    ElementTransformation & GetElement2Transformation();
    IntegrationPointTransformation & GetIntPoint1Transformation();
    IntegrationPointTransformation & GetIntPoint2Transformation();
+
+   /** @brief Check for self-consistency: compares the result of mapping the
+       reference face vertices to physical coordinates using the three
+       transformations: face, element 1, and element 2.
+
+       @param[in] print_level  If set to a positive number, print the physical
+                               coordinates of the face vertices computed through
+                               all available transformations: face, element 1,
+                               and/or element 2.
+       @param[in,out] out      The output stream to use for printing.
+
+       @returns A maximal distance between physical coordinates of face vertices
+                that should coincide. A successful check should return a small
+                number relative to the mesh extents. If less than 2 of the three
+                transformations are set, returns 0.
+
+       @warning This check will generally fail on periodic boundary faces.
+   */
+   double CheckConsistency(int print_level = 0,
+                           std::ostream &out = mfem::out);
 };
 
-/*                 Elem1(Loc1(x)) = Face(x) = Elem2(Loc2(x))
+/**                Elem1(Loc1(x)) = Face(x) = Elem2(Loc2(x))
 
 
                                 Physical Space

fem/estimators.hpp

@@ -45,6 +45,7 @@ public:
    /// Force recomputation of the estimates on the next call to GetLocalErrors.
    virtual void Reset() = 0;
 
+   /// Destruct the error estimator
    virtual ~ErrorEstimator() { }
 };
 
@@ -66,6 +67,14 @@ public:
 /** @brief The ZienkiewiczZhuEstimator class implements the Zienkiewicz-Zhu
     error estimation procedure.
 
+    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    and a posteriori error estimates. Part 1: The recovery technique.
+    Int. J. Num. Meth. Engng. 33, 1331-1364 (1992).
+
+    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    and a posteriori error estimates. Part 2: Error estimates and adaptivity.
+    Int. J. Num. Meth. Engng. 33, 1365-1382 (1992).
+
     The required BilinearFormIntegrator must implement the methods
     ComputeElementFlux() and ComputeFluxEnergy().
  */
@@ -217,6 +226,7 @@ protected:
       class when needed.*/
    bool own_flux_fes; ///< Ownership flag for flux_space and smooth_flux_space.
 
+   /// Initialize with the integrator, solution, and flux finite element spaces.
    void Init(BilinearFormIntegrator &integ,
              ParGridFunction &sol,
              ParFiniteElementSpace *flux_fes,

fem/fe_coll.cpp

@@ -311,10 +311,10 @@ GetEdge(int &nv, v_t &v, int &ne, int &e, int &eo, const int edge_info)
    eo = edge_info%64;
    MFEM_ASSERT(0 <= e && e < g_consts::NumEdges, "");
    MFEM_ASSERT(0 <= eo && eo < e_consts::NumOrient, "");
-   v[0] = g_consts::Edges[e][0];
-   v[1] = g_consts::Edges[e][1];
-   v[0] = e_consts::Orient[eo][v[0]];
-   v[1] = e_consts::Orient[eo][v[1]];
+   v[0] = e_consts::Orient[eo][0];
+   v[1] = e_consts::Orient[eo][1];
+   v[0] = g_consts::Edges[e][v[0]];
+   v[1] = g_consts::Edges[e][v[1]];
 }
 
 template <Geometry::Type geom, Geometry::Type f_geom,

fem/fe_coll.hpp

@@ -19,10 +19,10 @@
 namespace mfem
 {
 
-/** Collection of finite elements from the same family in multiple dimensions.
-    This class is used to match the degrees of freedom of a FiniteElementSpace
-    between elements, and to provide the finite element restriction from an
-    element to its boundary. */
+/** @brief Collection of finite elements from the same family in multiple
+    dimensions. This class is used to match the degrees of freedom of a
+    FiniteElementSpace between elements, and to provide the finite element
+    restriction from an element to its boundary. */
 class FiniteElementCollection
 {
 protected:
@@ -41,8 +41,7 @@ protected:
 
 public:
    /** @brief Enumeration for ContType: defines the continuity of the field
-       across element interfaces.
-   */
+       across element interfaces. */
    enum { CONTINUOUS,   ///< Field is continuous across element interfaces
           TANGENTIAL,   ///< Tangential components of vector field
           NORMAL,       ///< Normal component of vector field
@@ -77,15 +76,81 @@ public:
 
    /** @brief Factory method: return a newly allocated FiniteElementCollection
        according to the given name. */
+   /**
+   | FEC Name | Space | Order | BasisType | FiniteElement::MapT | Notes |
+   | :------: | :---: | :---: | :-------: | :-----: | :---: |
+   | H1_[DIM]_[ORDER] | H1 | * | 1 | VALUE | H1 nodal elements |
+   | H1@[BTYPE]_[DIM]_[ORDER] | H1 | * | * | VALUE | H1 nodal elements |
+   | H1Pos_[DIM]_[ORDER] | H1 | * | 1 | VALUE | H1 nodal elements |
+   | H1Pos_Trace_[DIM]_[ORDER] | H^{1/2} | * | 2 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
+   | H1_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
+   | H1_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 | VALUE | H^{1/2}-conforming trace elements for H1 defined on the interface between mesh elements (faces,edges,vertices) |
+   | ND_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | Nedelec vector elements |
+   | ND@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | Nedelec vector elements |
+   | ND_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0  | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
+   | ND_Trace@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
+   | RT_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | Raviart-Thomas vector elements |
+   | RT@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | Raviart-Thomas vector elements |
+   | RT_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_ValTrace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
+   | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinous L2 elements |
+   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinous L2 elements |
+   | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinous L2 elements |
+   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinous L2 elements |
+   | DG_Iface_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
+   | DG_Iface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
+   | DG_IntIface_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
+   | DG_IntIface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
+   | NURBS[ORDER] | - | * | - | VALUE | Non-Uniform Rational B-Splines (NURBS) elements |
+   | LinearNonConf3D | - | 1 | 1 | VALUE | Piecewise-linear nonconforming finite elements in 3D |
+   | CrouzeixRaviart | - | - | - | - | Crouzeix-Raviart nonconforming elements in 2D |
+   | Local_[FENAME] | - | - | - | - | Special collection that builds a local version out of the FENAME collection |
+   |-|-|-|-|-|-|
+   | Linear | H1 | 1 | 1 | VALUE | Left in for backward compatibility, consider using H1_ |
+   | Quadratic | H1 | 2 | 1 | VALUE | Left in for backward compatibility, consider using H1_ |
+   | QuadraticPos | H1 | 2 | 2 | VALUE | Left in for backward compatibility, consider using H1_ |
+   | Cubic | H1 | 2 | 1 | VALUE | Left in for backward compatibility, consider using H1_ |
+   | Const2D | L2 | 0 | 1 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | Const3D | L2 | 0 | 1 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | LinearDiscont2D | L2 | 1 | 1 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | GaussLinearDiscont2D | L2 | 1 | 0 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | P1OnQuad | H1 | 1 | 1 | VALUE | Linear P1 element with 3 nodes on a square |
+   | QuadraticDiscont2D | L2 | 2 | 1 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | QuadraticPosDiscont2D | L2 | 2 | 2 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | GaussQuadraticDiscont2D | L2 | 2 | 0 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | CubicDiscont2D | L2 | 3 | 1 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | LinearDiscont3D | L2 | 1 | 1 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | QuadraticDiscont3D | L2 | 2 | 1 | VALUE | Left in for backward compatibility, consider using L2_ |
+   | ND1_3D | H(Curl) | 1 | 1 / 0 | H_CURL | Left in for backward compatibility, consider using ND_ |
+   | RT0_2D | H(Div) | 1 | 1 / 0 | H_DIV | Left in for backward compatibility, consider using RT_ |
+   | RT1_2D | H(Div) | 2 | 1 / 0 | H_DIV | Left in for backward compatibility, consider using RT_ |
+   | RT2_2D | H(Div) | 3 | 1 / 0 | H_DIV | Left in for backward compatibility, consider using RT_ |
+   | RT0_3D | H(Div) | 1 | 1 / 0 | H_DIV | Left in for backward compatibility, consider using RT_ |
+   | RT1_3D | H(Div) | 2 | 1 / 0 | H_DIV | Left in for backward compatibility, consider using RT_ |
+
+   | Tag | Description |
+   | :------: | :--------: |
+   | [DIM]    | Dimension of the elements (1D, 2D, 3D) |
+   | [ORDER]  | Approximation order of the elements (P0, P1, P2, ...) |
+   | [BTYPE]  | BasisType of the element (0-GaussLegendre, 1 - GaussLobatto, 2-Bernstein, 3-OpenUniform, 4-CloseUniform, 5-OpenHalfUniform) |
+   | [OBTYPE] | Open BasisType of the element for elements which have both types |
+   | [CBTYPE] | Closed BasisType of the element for elements which have both types |
+
+   [FENAME]  Is a special case for the Local FEC which generates a local version of a given
+   FEC.  It is selected from one of (BiCubic2DFiniteElement, Quad_Q3, Nedelec1HexFiniteElement,
+      Hex_ND1, H1_[DIM]_[ORDER],H1Pos_[DIM]_[ORDER], L2_[DIM]_[ORDER] )
+   */
    static FiniteElementCollection *New(const char *name);
 
    /** @brief Get the local dofs for a given sub-manifold.
 
-      Return the local dofs for a SDim-dimensional sub-manifold (0D - vertex,
-      1D - edge, 2D - face) including those on its boundary. The local index of
-      the sub-manifold (inside Geom) and its orientation are given by the
-      parameter Info = 64 * SubIndex + SubOrientation. Naturally, it is assumed
-      that 0 <= SDim <= Dim(Geom). */
+      Return the local dofs for a SDim-dimensional sub-manifold (0D - vertex, 1D
+      - edge, 2D - face) including those on its boundary. The local index of the
+      sub-manifold (inside Geom) and its orientation are given by the parameter
+      Info = 64 * SubIndex + SubOrientation. Naturally, it is assumed that 0 <=
+      SDim <= Dim(Geom). */
    void SubDofOrder(Geometry::Type Geom, int SDim, int Info,
                     Array<int> &dofs) const;
 };
@@ -123,8 +188,8 @@ public:
    virtual ~H1_FECollection();
 };
 
-/** Arbitrary order H1-conforming (continuous) finite elements with positive
-    basis functions. */
+/** @brief Arbitrary order H1-conforming (continuous) finite elements with
+    positive basis functions. */
 class H1Pos_FECollection : public H1_FECollection
 {
 public:
@@ -132,6 +197,7 @@ public:
       : H1_FECollection(p, dim, BasisType::Positive) { }
 };
 
+
 /** Arbitrary order H1-conforming (continuous) serendipity finite elements;
     Current implementation works in 2D only; 3D version is in development. */
 class H1Ser_FECollection : public H1_FECollection
@@ -141,9 +207,9 @@ public:
       : H1_FECollection(p, dim, BasisType::Serendipity) { };
 };
 
-/** Arbitrary order "H^{1/2}-conforming" trace finite elements defined on the
-    interface between mesh elements (faces,edges,vertices); these are the trace
-    FEs of the H1-conforming FEs. */
+/** @brief Arbitrary order "H^{1/2}-conforming" trace finite elements defined on
+    the interface between mesh elements (faces,edges,vertices); these are the
+    trace FEs of the H1-conforming FEs. */
 class H1_Trace_FECollection : public H1_FECollection
 {
 public:
@@ -241,9 +307,9 @@ public:
    virtual ~RT_FECollection();
 };
 
-/** Arbitrary order "H^{-1/2}-conforming" face finite elements defined on the
-    interface between mesh elements (faces); these are the normal trace FEs of
-    the H(div)-conforming FEs. */
+/** @brief Arbitrary order "H^{-1/2}-conforming" face finite elements defined on
+    the interface between mesh elements (faces); these are the normal trace FEs
+    of the H(div)-conforming FEs. */
 class RT_Trace_FECollection : public RT_FECollection
 {
 public:
@@ -291,9 +357,9 @@ public:
    virtual ~ND_FECollection();
 };
 
-/** Arbitrary order H(curl)-trace finite elements defined on the interface
-    between mesh elements (faces,edges); these are the tangential trace FEs of
-    the H(curl)-conforming FEs. */
+/** @brief Arbitrary order H(curl)-trace finite elements defined on the
+    interface between mesh elements (faces,edges); these are the tangential
+    trace FEs of the H(curl)-conforming FEs. */
 class ND_Trace_FECollection : public ND_FECollection
 {
 public:
@@ -358,7 +424,7 @@ public:
 };
 
 
-/// Piecewise-(bi)linear continuous finite elements.
+/// Piecewise-(bi/tri)linear continuous finite elements.
 class LinearFECollection : public FiniteElementCollection
 {
 private:
@@ -514,8 +580,8 @@ public:
 };
 
 
-/** First order Raviart-Thomas finite elements in 2D. This class is kept only
-    for backward compatibility, consider using RT_FECollection instead. */
+/** @brief First order Raviart-Thomas finite elements in 2D. This class is kept
+    only for backward compatibility, consider using RT_FECollection instead. */
 class RT0_2DFECollection : public FiniteElementCollection
 {
 private:
@@ -538,8 +604,8 @@ public:
    virtual int GetContType() const { return NORMAL; }
 };
 
-/** Second order Raviart-Thomas finite elements in 2D. This class is kept only
-    for backward compatibility, consider using RT_FECollection instead. */
+/** @brief Second order Raviart-Thomas finite elements in 2D. This class is kept
+    only for backward compatibility, consider using RT_FECollection instead. */
 class RT1_2DFECollection : public FiniteElementCollection
 {
 private:
@@ -562,8 +628,8 @@ public:
    virtual int GetContType() const { return NORMAL; }
 };
 
-/** Third order Raviart-Thomas finite elements in 2D. This class is kept only
-    for backward compatibility, consider using RT_FECollection instead. */
+/** @brief Third order Raviart-Thomas finite elements in 2D. This class is kept
+    only for backward compatibility, consider using RT_FECollection instead. */
 class RT2_2DFECollection : public FiniteElementCollection
 {
 private:
@@ -586,8 +652,9 @@ public:
    virtual int GetContType() const { return NORMAL; }
 };
 
-/** Piecewise-constant discontinuous finite elements in 2D. This class is kept
-    only for backward compatibility, consider using L2_FECollection instead. */
+/** @brief Piecewise-constant discontinuous finite elements in 2D. This class is
+    kept only for backward compatibility, consider using L2_FECollection
+    instead. */
 class Const2DFECollection : public FiniteElementCollection
 {
 private:
@@ -609,8 +676,9 @@ public:
    virtual int GetContType() const { return DISCONTINUOUS; }
 };
 
-/** Piecewise-linear discontinuous finite elements in 2D. This class is kept
-    only for backward compatibility, consider using L2_FECollection instead. */
+/** @brief Piecewise-linear discontinuous finite elements in 2D. This class is
+    kept only for backward compatibility, consider using L2_FECollection
+    instead. */
 class LinearDiscont2DFECollection : public FiniteElementCollection
 {
 private:
@@ -673,8 +741,9 @@ public:
    virtual int GetContType() const { return DISCONTINUOUS; }
 };
 
-/** Piecewise-quadratic discontinuous finite elements in 2D. This class is kept
-    only for backward compatibility, consider using L2_FECollection instead. */
+/** @brief Piecewise-quadratic discontinuous finite elements in 2D. This class
+    is kept only for backward compatibility, consider using L2_FECollection
+    instead. */
 class QuadraticDiscont2DFECollection : public FiniteElementCollection
 {
 private:
@@ -737,8 +806,9 @@ public:
    virtual int GetContType() const { return DISCONTINUOUS; }
 };
 
-/** Piecewise-cubic discontinuous finite elements in 2D. This class is kept
-    only for backward compatibility, consider using L2_FECollection instead. */
+/** @brief Piecewise-cubic discontinuous finite elements in 2D. This class is
+    kept only for backward compatibility, consider using L2_FECollection
+    instead. */
 class CubicDiscont2DFECollection : public FiniteElementCollection
 {
 private:
@@ -760,8 +830,9 @@ public:
    virtual int GetContType() const { return DISCONTINUOUS; }
 };
 
-/** Piecewise-constant discontinuous finite elements in 3D. This class is kept
-    only for backward compatibility, consider using L2_FECollection instead. */
+/** @brief Piecewise-constant discontinuous finite elements in 3D. This class is
+    kept only for backward compatibility, consider using L2_FECollection
+    instead. */
 class Const3DFECollection : public FiniteElementCollection
 {
 private:
@@ -784,8 +855,9 @@ public:
    virtual int GetContType() const { return DISCONTINUOUS; }
 };
 
-/** Piecewise-linear discontinuous finite elements in 3D. This class is kept
-    only for backward compatibility, consider using L2_FECollection instead. */
+/** @brief Piecewise-linear discontinuous finite elements in 3D. This class is
+    kept only for backward compatibility, consider using L2_FECollection
+    instead. */
 class LinearDiscont3DFECollection : public FiniteElementCollection
 {
 private:
@@ -807,8 +879,9 @@ public:
    virtual int GetContType() const { return DISCONTINUOUS; }
 };
 
-/** Piecewise-quadratic discontinuous finite elements in 3D. This class is kept
-    only for backward compatibility, consider using L2_FECollection instead. */
+/** @brief Piecewise-quadratic discontinuous finite elements in 3D. This class
+    is kept only for backward compatibility, consider using L2_FECollection
+    instead. */
 class QuadraticDiscont3DFECollection : public FiniteElementCollection
 {
 private:
@@ -856,8 +929,9 @@ public:
    virtual int GetContType() const { return CONTINUOUS; }
 };
 
-/** Lowest order Nedelec finite elements in 3D. This class is kept only for
-    backward compatibility, consider using the new ND_FECollection instead. */
+/** @brief Lowest order Nedelec finite elements in 3D. This class is kept only
+    for backward compatibility, consider using the new ND_FECollection
+    instead. */
 class ND1_3DFECollection : public FiniteElementCollection
 {
 private:
@@ -879,8 +953,8 @@ public:
    virtual int GetContType() const { return TANGENTIAL; }
 };
 
-/** First order Raviart-Thomas finite elements in 3D. This class is kept only
-    for backward compatibility, consider using RT_FECollection instead. */
+/** @brief First order Raviart-Thomas finite elements in 3D. This class is kept
+    only for backward compatibility, consider using RT_FECollection instead. */
 class RT0_3DFECollection : public FiniteElementCollection
 {
 private:
@@ -903,8 +977,8 @@ public:
    virtual int GetContType() const { return NORMAL; }
 };
 
-/** Second order Raviart-Thomas finite elements in 3D. This class is kept only
-    for backward compatibility, consider using RT_FECollection instead. */
+/** @brief Second order Raviart-Thomas finite elements in 3D. This class is kept
+    only for backward compatibility, consider using RT_FECollection instead. */
 class RT1_3DFECollection : public FiniteElementCollection
 {
 private:

fem/fespace.cpp

@@ -60,7 +60,7 @@ FiniteElementSpace::FiniteElementSpace()
    : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
      ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
      fdofs(NULL), bdofs(NULL),
-     elem_dof(NULL), bdrElem_dof(NULL),
+     elem_dof(NULL), bdrElem_dof(NULL), face_dof(NULL),
      NURBSext(NULL), own_ext(false),
      cP(NULL), cR(NULL), cP_is_set(false),
      Th(Operator::ANY_TYPE),
@@ -233,6 +233,54 @@ void FiniteElementSpace::BuildElementToDofTable() const
    elem_dof = el_dof;
 }
 
+void FiniteElementSpace::BuildBdrElementToDofTable() const
+{
+   if (bdrElem_dof) { return; }
+
+   Table *bel_dof = new Table;
+   Array<int> dofs;
+   bel_dof->MakeI(mesh->GetNBE());
+   for (int i = 0; i < mesh->GetNBE(); i++)
+   {
+      GetBdrElementDofs(i, dofs);
+      bel_dof->AddColumnsInRow(i, dofs.Size());
+   }
+   bel_dof->MakeJ();
+   for (int i = 0; i < mesh->GetNBE(); i++)
+   {
+      GetBdrElementDofs(i, dofs);
+      bel_dof->AddConnections(i, (int *)dofs, dofs.Size());
+   }
+   bel_dof->ShiftUpI();
+   bdrElem_dof = bel_dof;
+}
+
+void FiniteElementSpace::BuildFaceToDofTable() const
+{
+   // Here, "face" == (dim-1)-dimensional mesh entity.
+
+   if (face_dof) { return; }
+
+   if (NURBSext) { BuildNURBSFaceToDofTable(); return; }
+
+   Table *fc_dof = new Table;
+   Array<int> dofs;
+   fc_dof->MakeI(mesh->GetNumFaces());
+   for (int i = 0; i < fc_dof->Size(); i++)
+   {
+      GetFaceDofs(i, dofs);
+      fc_dof->AddColumnsInRow(i, dofs.Size());
+   }
+   fc_dof->MakeJ();
+   for (int i = 0; i < fc_dof->Size(); i++)
+   {
+      GetFaceDofs(i, dofs);
+      fc_dof->AddConnections(i, (int *)dofs, dofs.Size());
+   }
+   fc_dof->ShiftUpI();
+   face_dof = fc_dof;
+}
+
 void FiniteElementSpace::RebuildElementToDofTable()
 {
    delete elem_dof;
@@ -1456,6 +1504,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    this->ordering = (Ordering::Type) ordering;
 
    elem_dof = NULL;
+   face_dof = NULL;
    sequence = mesh->GetSequence();
    Th.SetType(Operator::ANY_TYPE);
 
@@ -1505,6 +1554,8 @@ NURBSExtension *FiniteElementSpace::StealNURBSext()
 
 void FiniteElementSpace::UpdateNURBS()
 {
+   MFEM_VERIFY(NURBSext, "NURBSExt not defined.");
+
    nvdofs = 0;
    nedofs = 0;
    nfdofs = 0;
@@ -1512,6 +1563,10 @@ void FiniteElementSpace::UpdateNURBS()
    fdofs = NULL;
    bdofs = NULL;
 
+   delete face_dof;
+   face_dof = NULL;
+   face_to_be.DeleteAll();
+
    dynamic_cast<const NURBSFECollection *>(fec)->Reset();
 
    ndofs = NURBSext->GetNDof();
@@ -1519,6 +1574,55 @@ void FiniteElementSpace::UpdateNURBS()
    bdrElem_dof = NURBSext->GetBdrElementDofTable();
 }
 
+void FiniteElementSpace::BuildNURBSFaceToDofTable() const
+{
+   if (face_dof) { return; }
+
+   const int dim = mesh->Dimension();
+
+   // Find bdr to face mapping
+   face_to_be.SetSize(GetNF());
+   face_to_be = -1;
+   for (int b = 0; b < GetNBE(); b++)
+   {
+      int f = mesh->GetBdrElementEdgeIndex(b);
+      face_to_be[f] = b;
+   }
+
+   // Loop over faces in correct order, to prevent a sort
+   // Sort will destroy orientation info in ordering of dofs
+   Array<Connection> face_dof_list;
+   Array<int> row;
+   for (int f = 0; f < GetNF(); f++)
+   {
+      int b = face_to_be[f];
+      if (b == -1) { continue; }
+      // FIXME: this assumes the boundary element and the face element have the
+      //        same orientation.
+      if (dim > 1)
+      {
+         const Element *fe = mesh->GetFace(f);
+         const Element *be = mesh->GetBdrElement(b);
+         const int nv = be->GetNVertices();
+         const int *fv = fe->GetVertices();
+         const int *bv = be->GetVertices();
+         for (int i = 0; i < nv; i++)
+         {
+            MFEM_VERIFY(fv[i] == bv[i],
+                        "non-matching face and boundary elements detected!");
+         }
+      }
+      GetBdrElementDofs(b, row);
+      Connection conn(f,0);
+      for (int i = 0; i < row.Size(); i++)
+      {
+         conn.to = row[i];
+         face_dof_list.Append(conn);
+      }
+   }
+   face_dof = new Table(GetNF(), face_dof_list);
+}
+
 void FiniteElementSpace::Construct()
 {
    // This method should be used only for non-NURBS spaces.
@@ -1526,6 +1630,7 @@ void FiniteElementSpace::Construct()
 
    elem_dof = NULL;
    bdrElem_dof = NULL;
+   face_dof = NULL;
 
    ndofs = 0;
    nedofs = nfdofs = nbdofs = 0;
@@ -1788,59 +1893,68 @@ void FiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
 
 void FiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs) const
 {
-   int j, k, nv, ne, nf, nd, dim = mesh->Dimension();
-   Array<int> V, E, Eo;
-   const int *ind;
+   // If face_dof is already built, use it.
+   // If it is not and we have a NURBS space, build the face_dof and use it.
+   if (face_dof || (NURBSext && (BuildNURBSFaceToDofTable(), true)))
+   {
+      face_dof->GetRow(i, dofs);
+   }
+   else
+   {
+      int j, k, nv, ne, nf, nd, dim = mesh->Dimension();
+      Array<int> V, E, Eo;
+      const int *ind;
 
-   // for 1D, 2D and 3D faces
-   nv = fec->DofForGeometry(Geometry::POINT);
-   ne = (dim > 1) ? fec->DofForGeometry(Geometry::SEGMENT) : 0;
-   if (nv > 0)
-   {
-      mesh->GetFaceVertices(i, V);
-   }
-   if (ne > 0)
-   {
-      mesh->GetFaceEdges(i, E, Eo);
-   }
-   nf = (fdofs) ? (fdofs[i+1]-fdofs[i]) : (0);
-   nd = V.Size() * nv + E.Size() * ne + nf;
-   dofs.SetSize(nd);
-   if (nv > 0)
-   {
-      for (k = 0; k < V.Size(); k++)
+      // for 1D, 2D and 3D faces
+      nv = fec->DofForGeometry(Geometry::POINT);
+      ne = (dim > 1) ? fec->DofForGeometry(Geometry::SEGMENT) : 0;
+      if (nv > 0)
       {
-         for (j = 0; j < nv; j++)
-         {
-            dofs[k*nv+j] = V[k]*nv+j;
-         }
+         mesh->GetFaceVertices(i, V);
       }
-   }
-   nv *= V.Size();
-   if (ne > 0)
-   {
-      for (k = 0; k < E.Size(); k++)
+      if (ne > 0)
       {
-         ind = fec->DofOrderForOrientation(Geometry::SEGMENT, Eo[k]);
-         for (j = 0; j < ne; j++)
+         mesh->GetFaceEdges(i, E, Eo);
+      }
+      nf = (fdofs) ? (fdofs[i+1]-fdofs[i]) : (0);
+      nd = V.Size() * nv + E.Size() * ne + nf;
+      dofs.SetSize(nd);
+      if (nv > 0)
+      {
+         for (k = 0; k < V.Size(); k++)
          {
-            if (ind[j] < 0)
+            for (j = 0; j < nv; j++)
             {
-               dofs[nv+k*ne+j] = -1 - ( nvdofs+E[k]*ne+(-1-ind[j]) );
-            }
-            else
-            {
-               dofs[nv+k*ne+j] = nvdofs+E[k]*ne+ind[j];
+               dofs[k*nv+j] = V[k]*nv+j;
             }
          }
       }
-   }
-   ne = nv + ne * E.Size();
-   if (nf > 0)
-   {
-      for (j = nvdofs+nedofs+fdofs[i], k = 0; k < nf; j++, k++)
+      nv *= V.Size();
+      if (ne > 0)
       {
-         dofs[ne+k] = j;
+         for (k = 0; k < E.Size(); k++)
+         {
+            ind = fec->DofOrderForOrientation(Geometry::SEGMENT, Eo[k]);
+            for (j = 0; j < ne; j++)
+            {
+               if (ind[j] < 0)
+               {
+                  dofs[nv+k*ne+j] = -1 - ( nvdofs+E[k]*ne+(-1-ind[j]) );
+               }
+               else
+               {
+                  dofs[nv+k*ne+j] = nvdofs+E[k]*ne+ind[j];
+               }
+            }
+         }
+      }
+      ne = nv + ne * E.Size();
+      if (nf > 0)
+      {
+         for (j = nvdofs+nedofs+fdofs[i], k = 0; k < nf; j++, k++)
+         {
+            dofs[ne+k] = j;
+         }
       }
    }
 }
@@ -1969,14 +2083,21 @@ const FiniteElement *FiniteElementSpace::GetFaceElement(int i) const
          fe = fec->FiniteElementForGeometry(mesh->GetFaceBaseGeometry(i));
    }
 
-   // if (NURBSext)
-   //    NURBSext->LoadFaceElement(i, fe);
+   if (NURBSext)
+   {
+      // Ensure 'face_to_be' is built:
+      if (!face_dof) { BuildNURBSFaceToDofTable(); }
+      MFEM_ASSERT(face_to_be[i] >= 0,
+                  "NURBS mesh: only boundary faces are supported!");
+      NURBSext->LoadBE(face_to_be[i], fe);
+   }
 
    return fe;
 }
 
 const FiniteElement *FiniteElementSpace::GetEdgeElement(int i) const
 {
+   MFEM_ASSERT(mesh->Dimension() > 1, "No edges with a mesh dimension < 2");
    return fec->FiniteElementForGeometry(Geometry::SEGMENT);
 }
 
@@ -2024,11 +2145,14 @@ void FiniteElementSpace::Destroy()
    if (NURBSext)
    {
       if (own_ext) { delete NURBSext; }
+      delete face_dof;
+      face_to_be.DeleteAll();
    }
    else
    {
       delete elem_dof;
       delete bdrElem_dof;
+      delete face_dof;
 
       delete [] bdofs;
       delete [] fdofs;

fem/fespace.hpp

@@ -111,7 +111,9 @@ protected:
    int *fdofs, *bdofs;
 
    mutable Table *elem_dof; // if NURBS FE space, not owned; otherwise, owned.
-   Table *bdrElem_dof; // used only with NURBS FE spaces; not owned.
+   mutable Table *bdrElem_dof; // not owned only if NURBS FE space.
+   mutable Table *face_dof; // owned
+   mutable Array<int> face_to_be; // used only with NURBS FE spaces; owned.
 
    Array<int> dof_elem_array, dof_ldof_array;
 
@@ -158,6 +160,14 @@ protected:
    void Destroy();
 
    void BuildElementToDofTable() const;
+   void BuildBdrElementToDofTable() const;
+   void BuildFaceToDofTable() const;
+
+   /** @brief  Generates partial face_dof table for a NURBS space.
+
+       The table is only defined for exterior faces that coincide with a
+       boundary. */
+   void BuildNURBSFaceToDofTable() const;
 
    /// Helpers to remove encoded sign from a DOF
    static inline int DecodeDof(int dof)
@@ -206,7 +216,7 @@ protected:
       virtual ~RefinementOperator();
    };
 
-   // Derefinement operator, used by the friend class InterpolationGridTransfer.
+   /// Derefinement operator, used by the friend class InterpolationGridTransfer.
    class DerefinementOperator : public Operator
    {
       const FiniteElementSpace *fine_fes; // Not owned.
@@ -225,12 +235,12 @@ protected:
       virtual ~DerefinementOperator();
    };
 
-   // This method makes the same assumptions as the method:
-   //    void GetLocalRefinementMatrices(
-   //       const FiniteElementSpace &coarse_fes, Geometry::Type geom,
-   //       DenseTensor &localP) const
-   // which is defined below. It also assumes that the coarse fes and this have
-   // the same vector dimension, vdim.
+   /** This method makes the same assumptions as the method:
+       void GetLocalRefinementMatrices(
+           const FiniteElementSpace &coarse_fes, Geometry::Type geom,
+           DenseTensor &localP) const
+       which is defined below. It also assumes that the coarse fes and this have
+       the same vector dimension, vdim. */
    SparseMatrix *RefinementMatrix_main(const int coarse_ndofs,
                                        const Table &coarse_elem_dof,
                                        const DenseTensor localP[]) const;
@@ -248,11 +258,13 @@ protected:
    /// Calculate GridFunction restriction matrix after mesh derefinement.
    SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof);
 
-   // This method assumes that this->mesh is a refinement of coarse_fes->mesh
-   // and that the CoarseFineTransformations of this->mesh are set accordingly.
-   // Another assumption is that the FEs of this use the same MapType as the FEs
-   // of coarse_fes. Finally, it assumes that the spaces this and coarse_fes are
-   // NOT variable-order spaces.
+   /** @brief Return in @a localP the local refinement matrices that map
+       between fespaces after mesh refinement. */
+   /** This method assumes that this->mesh is a refinement of coarse_fes->mesh
+       and that the CoarseFineTransformations of this->mesh are set accordingly.
+       Another assumption is that the FEs of this use the same MapType as the FEs
+       of coarse_fes. Finally, it assumes that the spaces this and coarse_fes are
+       NOT variable-order spaces. */
    void GetLocalRefinementMatrices(const FiniteElementSpace &coarse_fes,
                                    Geometry::Type geom,
                                    DenseTensor &localP) const;
@@ -467,11 +479,11 @@ public:
    /// Returns indexes of degrees of freedom for i'th boundary element.
    virtual void GetBdrElementDofs(int i, Array<int> &dofs) const;
 
-   /** Returns the indexes of the degrees of freedom for i'th face
+   /** @brief eturns the indexes of the degrees of freedom for i'th face
        including the dofs for the edges and the vertices of the face. */
    virtual void GetFaceDofs(int i, Array<int> &dofs) const;
 
-   /** Returns the indexes of the degrees of freedom for i'th edge
+   /** @brief Returns the indexes of the degrees of freedom for i'th edge
        including the dofs for the vertices of the edge. */
    void GetEdgeDofs(int i, Array<int> &dofs) const;
 
@@ -526,28 +538,59 @@ public:
        is preserved. */
    void ReorderElementToDofTable();
 
+   /** @brief Return a reference to the internal Table that stores the lists of
+       scalar dofs, for each mesh element, as returned by GetElementDofs(). */
+   const Table &GetElementToDofTable() const { return *elem_dof; }
+
+   /** @brief Return a reference to the internal Table that stores the lists of
+       scalar dofs, for each boundary mesh element, as returned by
+       GetBdrElementDofs(). */
+   const Table &GetBdrElementToDofTable() const
+   { if (!bdrElem_dof) { BuildBdrElementToDofTable(); } return *bdrElem_dof; }
+
+   /** @brief Return a reference to the internal Table that stores the lists of
+       scalar dofs, for each face in the mesh, as returned by GetFaceDofs(). In
+       this context, "face" refers to a (dim-1)-dimensional mesh entity. */
+   /** @note In the case of a NURBS space, the rows corresponding to interior
+       faces will be empty. */
+   const Table &GetFaceToDofTable() const
+   { if (!face_dof) { BuildFaceToDofTable(); } return *face_dof; }
+
+   /** @brief Initialize internal data that enables the use of the methods
+       GetElementForDof() and GetLocalDofForDof(). */
    void BuildDofToArrays();
 
-   const Table &GetElementToDofTable() const { return *elem_dof; }
-   const Table &GetBdrElementToDofTable() const { return *bdrElem_dof; }
-
+   /// Return the index of the first element that contains dof @a i.
+   /** This method can be called only after setup is performed using the method
+       BuildDofToArrays(). */
    int GetElementForDof(int i) const { return dof_elem_array[i]; }
+   /// Return the local dof index in the first element that contains dof @a i.
+   /** This method can be called only after setup is performed using the method
+       BuildDofToArrays(). */
    int GetLocalDofForDof(int i) const { return dof_ldof_array[i]; }
 
-   /// Returns pointer to the FiniteElement associated with i'th element.
+   /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
+        associated with i'th element in the mesh object. */
    const FiniteElement *GetFE(int i) const;
 
-   /// Returns pointer to the FiniteElement for the i'th boundary element.
+   /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
+        associated with i'th boundary face in the mesh object. */
    const FiniteElement *GetBE(int i) const;
 
+   /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
+        associated with i'th face in the mesh object.  Faces in this case refer
+        to the MESHDIM-1 primitive so in 2D they are segments and in 1D they are
+        points.*/
    const FiniteElement *GetFaceElement(int i) const;
 
+   /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
+        associated with i'th edge in the mesh object. */
    const FiniteElement *GetEdgeElement(int i) const;
 
    /// Return the trace element from element 'i' to the given 'geom_type'
    const FiniteElement *GetTraceElement(int i, Geometry::Type geom_type) const;
 
-   /** Mark degrees of freedom associated with boundary elements with
+   /** @brief Mark degrees of freedom associated with boundary elements with
        the specified boundary attributes (marked in 'bdr_attr_is_ess').
        For spaces with 'vdim' > 1, the 'component' parameter can be used
        to restricts the marked vDOFs to the specified component. */
@@ -555,7 +598,7 @@ public:
                                   Array<int> &ess_vdofs,
                                   int component = -1) const;
 
-   /** Get a list of essential true dofs, ess_tdof_list, corresponding to the
+   /** @brief Get a list of essential true dofs, ess_tdof_list, corresponding to the
        boundary attributes marked in the array bdr_attr_is_ess.
        For spaces with 'vdim' > 1, the 'component' parameter can be used
        to restricts the marked tDOFs to the specified component. */
@@ -566,19 +609,19 @@ public:
    /// Convert a Boolean marker array to a list containing all marked indices.
    static void MarkerToList(const Array<int> &marker, Array<int> &list);
 
-   /** Convert an array of indices (list) to a Boolean marker array where all
+   /** @brief Convert an array of indices (list) to a Boolean marker array where all
        indices in the list are marked with the given value and the rest are set
        to zero. */
    static void ListToMarker(const Array<int> &list, int marker_size,
                             Array<int> &marker, int mark_val = -1);
 
-   /** For a partially conforming FE space, convert a marker array (nonzero
+   /** @brief For a partially conforming FE space, convert a marker array (nonzero
        entries are true) on the partially conforming dofs to a marker array on
        the conforming dofs. A conforming dofs is marked iff at least one of its
        dependent dofs is marked. */
    void ConvertToConformingVDofs(const Array<int> &dofs, Array<int> &cdofs);
 
-   /** For a partially conforming FE space, convert a marker array (nonzero
+   /** @brief For a partially conforming FE space, convert a marker array (nonzero
        entries are true) on the conforming dofs to a marker array on the
        (partially conforming) dofs. A dof is marked iff it depends on a marked
        conforming dofs, where dependency is defined by the ConformingRestriction
@@ -586,15 +629,15 @@ public:
        conforming dof. */
    void ConvertFromConformingVDofs(const Array<int> &cdofs, Array<int> &dofs);
 
-   /** Generate the global restriction matrix from a discontinuous
+   /** @brief Generate the global restriction matrix from a discontinuous
        FE space to the continuous FE space of the same polynomial degree. */
    SparseMatrix *D2C_GlobalRestrictionMatrix(FiniteElementSpace *cfes);
 
-   /** Generate the global restriction matrix from a discontinuous
+   /** @brief Generate the global restriction matrix from a discontinuous
        FE space to the piecewise constant FE space. */
    SparseMatrix *D2Const_GlobalRestrictionMatrix(FiniteElementSpace *cfes);
 
-   /** Construct the restriction matrix from the FE space given by
+   /** @brief Construct the restriction matrix from the FE space given by
        (*this) to the lower degree FE space given by (*lfes) which
        is defined on the same mesh. */
    SparseMatrix *H2L_GlobalRestrictionMatrix(FiniteElementSpace *lfes);
@@ -631,7 +674,7 @@ public:
    virtual void GetTrueTransferOperator(const FiniteElementSpace &coarse_fes,
                                         OperatorHandle &T) const;
 
-   /** Reflect changes in the mesh: update number of DOFs, etc. Also, calculate
+   /** @brief Reflect changes in the mesh: update number of DOFs, etc. Also, calculate
        GridFunction transformation operator (unless want_transform is false).
        Safe to call multiple times, does nothing if space already up to date. */
    virtual void Update(bool want_transform = true);
@@ -669,6 +712,7 @@ public:
       return dynamic_cast<const L2_FECollection*>(fec) != NULL;
    }
 
+   /// Save finite element space to output stream @a out.
    void Save(std::ostream &out) const;
 
    /** @brief Read a FiniteElementSpace from a stream. The returned
@@ -712,6 +756,12 @@ public:
    /// Return the total number of quadrature points.
    int GetSize() const { return size; }
 
+   /// Returns the mesh
+   inline Mesh *GetMesh() const { return mesh; }
+
+   /// Returns number of elements in the mesh.
+   inline int GetNE() const { return mesh->GetNE(); }
+
    /// Get the IntegrationRule associated with mesh element @a idx.
    const IntegrationRule &GetElementIntRule(int idx) const
    { return *int_rule[mesh->GetElementBaseGeometry(idx)]; }

fem/geom.cpp

@@ -1344,15 +1344,14 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
             return NULL;
          }
          ir = FindInIntPts(Geom, Times-1);
-         if (ir == NULL)
+         if (ir) { return ir; }
+
+         ir = new IntegrationRule(Times-1);
+         for (int i = 1; i < Times; i++)
          {
-            ir = new IntegrationRule(Times-1);
-            for (int i = 1; i < Times; i++)
-            {
-               IntegrationPoint &ip = ir->IntPoint(i-1);
-               ip.x = double(i) / Times;
-               ip.y = ip.z = 0.0;
-            }
+            IntegrationPoint &ip = ir->IntPoint(i-1);
+            ip.x = double(i) / Times;
+            ip.y = ip.z = 0.0;
          }
       }
       break;
@@ -1364,18 +1363,17 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
             return NULL;
          }
          ir = FindInIntPts(Geom, ((Times-1)*(Times-2))/2);
-         if (ir == NULL)
-         {
-            ir = new IntegrationRule(((Times-1)*(Times-2))/2);
-            for (int k = 0, j = 1; j < Times-1; j++)
-               for (int i = 1; i < Times-j; i++, k++)
-               {
-                  IntegrationPoint &ip = ir->IntPoint(k);
-                  ip.x = double(i) / Times;
-                  ip.y = double(j) / Times;
-                  ip.z = 0.0;
-               }
-         }
+         if (ir) { return ir; }
+
+         ir = new IntegrationRule(((Times-1)*(Times-2))/2);
+         for (int k = 0, j = 1; j < Times-1; j++)
+            for (int i = 1; i < Times-j; i++, k++)
+            {
+               IntegrationPoint &ip = ir->IntPoint(k);
+               ip.x = double(i) / Times;
+               ip.y = double(j) / Times;
+               ip.z = 0.0;
+            }
       }
       break;
 
@@ -1386,18 +1384,17 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
             return NULL;
          }
          ir = FindInIntPts(Geom, (Times-1)*(Times-1));
-         if (ir == NULL)
-         {
-            ir = new IntegrationRule((Times-1)*(Times-1));
-            for (int k = 0, j = 1; j < Times; j++)
-               for (int i = 1; i < Times; i++, k++)
-               {
-                  IntegrationPoint &ip = ir->IntPoint(k);
-                  ip.x = double(i) / Times;
-                  ip.y = double(j) / Times;
-                  ip.z = 0.0;
-               }
-         }
+         if (ir) { return ir; }
+
+         ir = new IntegrationRule((Times-1)*(Times-1));
+         for (int k = 0, j = 1; j < Times; j++)
+            for (int i = 1; i < Times; i++, k++)
+            {
+               IntegrationPoint &ip = ir->IntPoint(k);
+               ip.x = double(i) / Times;
+               ip.y = double(j) / Times;
+               ip.z = 0.0;
+            }
       }
       break;
 
@@ -1405,10 +1402,121 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
          mfem_error("GeometryRefiner::RefineInterior(...)");
    }
 
-   if (ir) { IntPts[Geom].Append(ir); }
+   MFEM_ASSERT(ir != NULL, "Failed to construct the refined IntegrationRule.");
+   IntPts[Geom].Append(ir);
+
    return ir;
 }
 
+
+int GeometryRefiner::GetRefinementLevelFromPoints(Geometry::Type geom, int Npts)
+{
+   switch (geom)
+   {
+      case Geometry::POINT:
+      {
+         return -1;
+      }
+      case Geometry::SEGMENT:
+      {
+         return Npts -1;
+      }
+      case Geometry::TRIANGLE:
+      {
+         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
+         {
+            np = (n+1)*(n+2)/2;
+            if (np == Npts) { return n; }
+         }
+         return -1;
+      }
+      case Geometry::SQUARE:
+      {
+         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
+         {
+            np = (n+1)*(n+1);
+            if (np == Npts) { return n; }
+         }
+         return -1;
+      }
+      case Geometry::CUBE:
+      {
+         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
+         {
+            np = (n+1)*(n+1)*(n+1);
+            if (np == Npts) { return n; }
+         }
+         return -1;
+      }
+      case Geometry::TETRAHEDRON:
+      {
+         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
+         {
+            np = (n+3)*(n+2)*(n+1)/6;
+            if (np == Npts) { return n; }
+         }
+         return -1;
+      }
+      case Geometry::PRISM:
+      {
+         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
+         {
+            np = (n+1)*(n+1)*(n+2)/2;
+            if (np == Npts) { return n; }
+         }
+         return -1;
+      }
+      default:
+      {
+         mfem_error("Non existing Geometry.");
+      }
+   }
+
+   return -1;
+}
+
+
+int GeometryRefiner::GetRefinementLevelFromElems(Geometry::Type geom, int Nels)
+{
+   switch (geom)
+   {
+      case Geometry::POINT:
+      {
+         return -1;
+      }
+      case Geometry::SEGMENT:
+      {
+         return Nels;
+      }
+      case Geometry::TRIANGLE:
+      case Geometry::SQUARE:
+      {
+         for (int n = 0; (n < 15) && (n*n < Nels+1) ; n++)
+         {
+            if (n*n == Nels) { return n-1; }
+         }
+         return -1;
+      }
+      case Geometry::CUBE:
+      case Geometry::TETRAHEDRON:
+      case Geometry::PRISM:
+      {
+         for (int n = 0; (n < 15) && (n*n*n < Nels+1) ; n++)
+         {
+            if (n*n*n == Nels) { return n-1; }
+         }
+         return -1;
+      }
+      default:
+      {
+         mfem_error("Non existing Geometry.");
+      }
+   }
+
+   return -1;
+}
+
+
 GeometryRefiner GlobGeometryRefiner;
 
 }

fem/geom.hpp

@@ -273,6 +273,12 @@ public:
    /// @note This method always uses Quadrature1D::OpenUniform points.
    const IntegrationRule *RefineInterior(Geometry::Type Geom, int Times);
 
+   /// Get the Refinement level based on number of points
+   virtual int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);
+
+   /// Get the Refinement level based on number of elements
+   virtual int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);
+
    ~GeometryRefiner();
 };
 

fem/gridfunc.cpp

@@ -199,8 +199,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
    if (f != fes) { Destroy(); }
    fes = f;
    v.UseDevice(true);
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
-                    fes->GetVSize(), true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    sequence = fes->GetSequence();
 }
 
@@ -397,8 +396,16 @@ const
    fes->DofsToVDofs(vdim-1, dofs);
    Vector DofVal(dofs.Size()), LocVec;
    const FiniteElement *fe = fes->GetFE(i);
-   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-   fe->CalcShape(ip, DofVal);
+   if (fe->GetMapType() == FiniteElement::VALUE)
+   {
+      fe->CalcShape(ip, DofVal);
+   }
+   else
+   {
+      ElementTransformation *Tr = fes->GetElementTransformation(i);
+      Tr->SetIntPoint(&ip);
+      fe->CalcPhysShape(*Tr, DofVal);
+   }
    GetSubVector(dofs, LocVec);
 
    return (DofVal * LocVec);
@@ -415,10 +422,17 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
    GetSubVector(vdofs, loc_data);
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
       Vector shape(dof);
-      FElem->CalcShape(ip, shape);
+      if (FElem->GetMapType() == FiniteElement::VALUE)
+      {
+         FElem->CalcShape(ip, shape);
+      }
+      else
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(i);
+         Tr->SetIntPoint(&ip);
+         FElem->CalcPhysShape(*Tr, shape);
+      }
       int vdim = fes->GetVDim();
       val.SetSize(vdim);
       for (int k = 0; k < vdim; k++)
@@ -752,19 +766,21 @@ double GridFunction::GetValue(ElementTransformation &T,
             IntegrationPoint fip;
             be_to_bfe(FET->GetGeometryType(), o, ip, fip);
 
-            FET->SetIntPoint(&fip);
+            // Compute and set the point in element 1 from fip
+            FET->SetAllIntPoints(&fip);
             ElementTransformation & T1 = FET->GetElement1Transformation();
             return GetValue(T1, T1.GetIntPoint(), comp);
          }
-         break;
       }
+      break;
       case ElementTransformation::BDR_FACE:
       {
          FaceElementTransformations * FET =
             dynamic_cast<FaceElementTransformations *>(&T);
 
          // Evaluate in neighboring element for both continuous and
-         // discontinuous fields.
+         // discontinuous fields (the integration point in T1 should have
+         // already been set).
          ElementTransformation & T1 = FET->GetElement1Transformation();
          return GetValue(T1, T1.GetIntPoint(), comp);
       }
@@ -888,19 +904,21 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
             IntegrationPoint fip;
             be_to_bfe(FET->GetGeometryType(), o, ip, fip);
 
-            FET->SetIntPoint(&fip);
+            // Compute and set the point in element 1 from fip
+            FET->SetAllIntPoints(&fip);
             ElementTransformation & T1 = FET->GetElement1Transformation();
             return GetVectorValue(T1, T1.GetIntPoint(), val);
          }
-         break;
       }
+      break;
       case ElementTransformation::BDR_FACE:
       {
          FaceElementTransformations * FET =
             dynamic_cast<FaceElementTransformations *>(&T);
 
          // Evaluate in neighboring element for both continuous and
-         // discontinuous fields.
+         // discontinuous fields (the integration point in T1 should have
+         // already been set).
          ElementTransformation & T1 = FET->GetElement1Transformation();
          return GetVectorValue(T1, T1.GetIntPoint(), val);
       }
@@ -1031,13 +1049,13 @@ int GridFunction::GetFaceVectorValues(
    }
    if (di == 0)
    {
-      Transf = fes->GetMesh()->GetFaceElementTransformations(i, 4);
+      Transf = fes->GetMesh()->GetFaceElementTransformations(i, 5);
       Transf->Loc1.Transform(ir, eir);
       GetVectorValues(*Transf->Elem1, eir, vals, &tr);
    }
    else
    {
-      Transf = fes->GetMesh()->GetFaceElementTransformations(i, 8);
+      Transf = fes->GetMesh()->GetFaceElementTransformations(i, 10);
       Transf->Loc2.Transform(ir, eir);
       GetVectorValues(*Transf->Elem2, eir, vals, &tr);
    }
@@ -1338,107 +1356,262 @@ void GridFunction::GetVectorGradientHat(
    MultAtB(loc_data_mat, dshape, gh);
 }
 
-double GridFunction::GetDivergence(ElementTransformation &tr) const
+double GridFunction::GetDivergence(ElementTransformation &T) const
 {
-   double div_v;
-   int elNo = tr.ElementNo;
-   const FiniteElement *FElem = fes->GetFE(elNo);
-   if (FElem->GetRangeType() == FiniteElement::SCALAR)
+   switch (T.ElementType)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
-      DenseMatrix grad_hat;
-      GetVectorGradientHat(tr, grad_hat);
-      const DenseMatrix &Jinv = tr.InverseJacobian();
-      div_v = 0.0;
-      for (int i = 0; i < Jinv.Width(); i++)
+      case ElementTransformation::ELEMENT:
       {
-         for (int j = 0; j < Jinv.Height(); j++)
+         int elNo = T.ElementNo;
+         const FiniteElement *fe = fes->GetFE(elNo);
+         if (fe->GetRangeType() == FiniteElement::SCALAR)
          {
-            div_v += grad_hat(i, j) * Jinv(j, i);
+            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                        "invalid FE map type");
+            DenseMatrix grad_hat;
+            GetVectorGradientHat(T, grad_hat);
+            const DenseMatrix &Jinv = T.InverseJacobian();
+            double div_v = 0.0;
+            for (int i = 0; i < Jinv.Width(); i++)
+            {
+               for (int j = 0; j < Jinv.Height(); j++)
+               {
+                  div_v += grad_hat(i, j) * Jinv(j, i);
+               }
+            }
+            return div_v;
+         }
+         else
+         {
+            // Assuming RT-type space
+            Array<int> dofs;
+            fes->GetElementDofs(elNo, dofs);
+            Vector loc_data, divshape(fe->GetDof());
+            GetSubVector(dofs, loc_data);
+            fe->CalcDivShape(T.GetIntPoint(), divshape);
+            return (loc_data * divshape) / T.Weight();
          }
       }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to properly capture the derivative of the normal component
+         // of the field (as well as the transverse divergence of the
+         // tangential compoents) we must evaluate it in the neighboring
+         // element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         return GetDivergence(T1);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         return GetDivergence(T1);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetDivergence: Unsupported element type \""
+                    << T.ElementType << "\"");
+      }
    }
-   else
-   {
-      // Assuming RT-type space
-      Array<int> dofs;
-      fes->GetElementDofs(elNo, dofs);
-      Vector loc_data, divshape(FElem->GetDof());
-      GetSubVector(dofs, loc_data);
-      FElem->CalcDivShape(tr.GetIntPoint(), divshape);
-      div_v = (loc_data * divshape) / tr.Weight();
-   }
-   return div_v;
+   return 0.0; // never reached
 }
 
-void GridFunction::GetCurl(ElementTransformation &tr, Vector &curl) const
+void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
 {
-   int elNo = tr.ElementNo;
-   const FiniteElement *FElem = fes->GetFE(elNo);
-   if (FElem->GetRangeType() == FiniteElement::SCALAR)
+   switch (T.ElementType)
    {
-      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
-                  "invalid FE map type");
-      DenseMatrix grad_hat;
-      GetVectorGradientHat(tr, grad_hat);
-      const DenseMatrix &Jinv = tr.InverseJacobian();
-      DenseMatrix grad(grad_hat.Height(), Jinv.Width()); // vdim x FElem->Dim
-      Mult(grad_hat, Jinv, grad);
-      MFEM_ASSERT(grad.Height() == grad.Width(), "");
-      if (grad.Height() == 3)
+      case ElementTransformation::ELEMENT:
       {
-         curl.SetSize(3);
-         curl(0) = grad(2,1) - grad(1,2);
-         curl(1) = grad(0,2) - grad(2,0);
-         curl(2) = grad(1,0) - grad(0,1);
+         int elNo = T.ElementNo;
+         const FiniteElement *fe = fes->GetFE(elNo);
+         if (fe->GetRangeType() == FiniteElement::SCALAR)
+         {
+            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                        "invalid FE map type");
+            DenseMatrix grad_hat;
+            GetVectorGradientHat(T, grad_hat);
+            const DenseMatrix &Jinv = T.InverseJacobian();
+            // Dimensions of grad are vdim x FElem->Dim
+            DenseMatrix grad(grad_hat.Height(), Jinv.Width());
+            Mult(grad_hat, Jinv, grad);
+            MFEM_ASSERT(grad.Height() == grad.Width(), "");
+            if (grad.Height() == 3)
+            {
+               curl.SetSize(3);
+               curl(0) = grad(2,1) - grad(1,2);
+               curl(1) = grad(0,2) - grad(2,0);
+               curl(2) = grad(1,0) - grad(0,1);
+            }
+            else if (grad.Height() == 2)
+            {
+               curl.SetSize(1);
+               curl(0) = grad(1,0) - grad(0,1);
+            }
+         }
+         else
+         {
+            // Assuming ND-type space
+            Array<int> dofs;
+            fes->GetElementDofs(elNo, dofs);
+            Vector loc_data;
+            GetSubVector(dofs, loc_data);
+            DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
+            fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
+            curl.SetSize(curl_shape.Width());
+            if (curl_shape.Width() == 3)
+            {
+               double curl_hat[3];
+               curl_shape.MultTranspose(loc_data, curl_hat);
+               T.Jacobian().Mult(curl_hat, curl);
+            }
+            else
+            {
+               curl_shape.MultTranspose(loc_data, curl);
+            }
+            curl /= T.Weight();
+         }
       }
-      else if (grad.Height() == 2)
+      break;
+      case ElementTransformation::BDR_ELEMENT:
       {
-         curl.SetSize(1);
-         curl(0) = grad(1,0) - grad(0,1);
+         // In order to capture the tangential components of the curl we
+         // must evaluate it in the neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetCurl(T1, curl);
       }
-   }
-   else
-   {
-      // Assuming ND-type space
-      Array<int> dofs;
-      fes->GetElementDofs(elNo, dofs);
-      Vector loc_data;
-      GetSubVector(dofs, loc_data);
-      DenseMatrix curl_shape(FElem->GetDof(), FElem->GetDim() == 3 ? 3 : 1);
-      FElem->CalcCurlShape(tr.GetIntPoint(), curl_shape);
-      curl.SetSize(curl_shape.Width());
-      if (curl_shape.Width() == 3)
+      break;
+      case ElementTransformation::BDR_FACE:
       {
-         double curl_hat[3];
-         curl_shape.MultTranspose(loc_data, curl_hat);
-         tr.Jacobian().Mult(curl_hat, curl);
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetCurl(T1, curl);
       }
-      else
+      break;
+      default:
       {
-         curl_shape.MultTranspose(loc_data, curl);
+         MFEM_ABORT("GridFunction::GetCurl: Unsupported element type \""
+                    << T.ElementType << "\"");
       }
-      curl /= tr.Weight();
    }
 }
 
-void GridFunction::GetGradient(ElementTransformation &tr, Vector &grad) const
+void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
 {
-   int elNo = tr.ElementNo;
-   const FiniteElement *fe = fes->GetFE(elNo);
-   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
-   int dim = fe->GetDim(), dof = fe->GetDof();
-   DenseMatrix dshape(dof, dim);
-   Vector lval, gh(dim);
-   Array<int> dofs;
+   switch (T.ElementType)
+   {
+      case ElementTransformation::ELEMENT:
+      {
+         const FiniteElement * fe = fes->GetFE(T.ElementNo);
+         MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
+                     "invalid FE map type");
+         int spaceDim = fes->GetMesh()->SpaceDimension();
+         int dim = fe->GetDim(), dof = fe->GetDof();
+         DenseMatrix dshape(dof, dim);
+         Vector lval, gh(dim);
+         Array<int> dofs;
 
-   grad.SetSize(dim);
-   fes->GetElementDofs(elNo, dofs);
-   GetSubVector(dofs, lval);
-   fe->CalcDShape(tr.GetIntPoint(), dshape);
-   dshape.MultTranspose(lval, gh);
-   tr.InverseJacobian().MultTranspose(gh, grad);
+         grad.SetSize(spaceDim);
+         fes->GetElementDofs(T.ElementNo, dofs);
+         GetSubVector(dofs, lval);
+         fe->CalcDShape(T.GetIntPoint(), dshape);
+         dshape.MultTranspose(lval, gh);
+         T.InverseJacobian().MultTranspose(gh, grad);
+      }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to properly capture the normal component of the gradient
+         // as well as its tangential components we must evaluate it in the
+         // neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetGradient(T1, grad);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetGradient(T1, grad);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetGradient: Unsupported element type \""
+                    << T.ElementType << "\"");
+      }
+   }
 }
 
 void GridFunction::GetGradients(ElementTransformation &tr,
@@ -1467,15 +1640,65 @@ void GridFunction::GetGradients(ElementTransformation &tr,
 }
 
 void GridFunction::GetVectorGradient(
-   ElementTransformation &tr, DenseMatrix &grad) const
+   ElementTransformation &T, DenseMatrix &grad) const
 {
-   MFEM_ASSERT(fes->GetFE(tr.ElementNo)->GetMapType() == FiniteElement::VALUE,
-               "invalid FE map type");
-   DenseMatrix grad_hat;
-   GetVectorGradientHat(tr, grad_hat);
-   const DenseMatrix &Jinv = tr.InverseJacobian();
-   grad.SetSize(grad_hat.Height(), Jinv.Width());
-   Mult(grad_hat, Jinv, grad);
+   switch (T.ElementType)
+   {
+      case ElementTransformation::ELEMENT:
+      {
+         MFEM_ASSERT(fes->GetFE(T.ElementNo)->GetMapType() ==
+                     FiniteElement::VALUE, "invalid FE map type");
+         DenseMatrix grad_hat;
+         GetVectorGradientHat(T, grad_hat);
+         const DenseMatrix &Jinv = T.InverseJacobian();
+         grad.SetSize(grad_hat.Height(), Jinv.Width());
+         Mult(grad_hat, Jinv, grad);
+      }
+      break;
+      case ElementTransformation::BDR_ELEMENT:
+      {
+         // In order to capture the normal component of the gradient we
+         // must evaluate it in the neighboring element.
+         FaceElementTransformations * FET =
+            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
+
+         // Boundary elements and Boundary Faces may have different
+         // orientations so adjust the integration point if necessary.
+         int o = 0;
+         if (fes->GetMesh()->Dimension() == 3)
+         {
+            int f;
+            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
+         }
+
+         IntegrationPoint fip;
+         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
+
+         // Compute and set the point in element 1 from fip
+         FET->SetAllIntPoints(&fip);
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+
+         GetVectorGradient(T1, grad);
+      }
+      break;
+      case ElementTransformation::BDR_FACE:
+      {
+         // This must be a DG context so this dynamic cast must succeed.
+         FaceElementTransformations * FET =
+            dynamic_cast<FaceElementTransformations *>(&T);
+
+         // Evaluate in neighboring element (the integration point in T1 should
+         // have already been set).
+         ElementTransformation & T1 = FET->GetElement1Transformation();
+         GetVectorGradient(T1, grad);
+      }
+      break;
+      default:
+      {
+         MFEM_ABORT("GridFunction::GetVectorGradient: "
+                    "Unsupported element type \"" << T.ElementType << "\"");
+      }
+   }
 }
 
 void GridFunction::GetElementAverages(GridFunction &avgs) const

fem/gridfunc.hpp

@@ -105,7 +105,7 @@ public:
        have the same size.
 
        @note Defining this method overwrites the implicitly defined copy
-       assignemnt operator. */
+       assignment operator. */
    GridFunction &operator=(const GridFunction &rhs)
    { return operator=((const Vector &)rhs); }
 
@@ -162,7 +162,8 @@ public:
                            int vdim = 1) const;
 
    /** Return a vector value from within the given element. */
-   void GetVectorValue(int i, const IntegrationPoint &ip, Vector &val) const;
+   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
+                               Vector &val) const;
    ///@}
 
    /** @name Element Index Get Values Methods
@@ -208,13 +209,14 @@ public:
    ///@{
    /** Return a scalar value from within the element indicated by the
        ElementTransformation Object. */
-   double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
-                   int comp = 0, Vector *tr = NULL) const;
+   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
+                           int comp = 0, Vector *tr = NULL) const;
 
    /** Return a vector value from within the element indicated by the
        ElementTransformation Object. */
-   void GetVectorValue(ElementTransformation &T, const IntegrationPoint &ip,
-                       Vector &val, Vector *tr = NULL) const;
+   virtual void GetVectorValue(ElementTransformation &T,
+                               const IntegrationPoint &ip,
+                               Vector &val, Vector *tr = NULL) const;
    ///@}
 
    /** @name ElementTransformation Get Values Methods
@@ -598,11 +600,13 @@ public:
                      type = adios2stream::data_type::point_data) const;
 #endif
 
-   /** Write the GridFunction in VTK format. Note that Mesh::PrintVTK must be
-       called first. The parameter ref > 0 must match the one used in
+   /** @brief Write the GridFunction in VTK format. Note that Mesh::PrintVTK
+       must be called first. The parameter ref > 0 must match the one used in
        Mesh::PrintVTK. */
    void SaveVTK(std::ostream &out, const std::string &field_name, int ref);
 
+   /** @brief Write the GridFunction in STL format. Note that the mesh dimension
+       must be 2 and that quad elements will be broken into two triangles.*/
    void SaveSTL(std::ostream &out, int TimesToRefine = 1);
 
    /// Destroys grid function.
@@ -710,7 +714,7 @@ public:
        the same size.
 
        @note Defining this method overwrites the implicitly defined copy
-       assignemnt operator. */
+       assignment operator. */
    QuadratureFunction &operator=(const QuadratureFunction &v);
 
    /// Get the IntegrationRule associated with mesh element @a idx.

fem/gslib.cpp

@@ -192,6 +192,7 @@ void FindPointsGSLIB::Interpolate(Array<unsigned int> &codes,
    const int ncomp      = field_in.FESpace()->GetVDim(),
              points_fld = field_in.Size() / ncomp,
              points_cnt = codes.Size();
+   field_out.SetSize(points_cnt*ncomp);
 
    for (int i = 0; i < ncomp; i++)
    {

fem/intrules.cpp

@@ -618,6 +618,26 @@ void QuadratureFunctions1D::OpenHalfUniform(const int np, IntegrationRule* ir)
    CalculateUniformWeights(ir, Quadrature1D::OpenHalfUniform);
 }
 
+void QuadratureFunctions1D::ClosedGL(const int np, IntegrationRule* ir)
+{
+   ir->SetSize(np);
+   ir->IntPoint(0).x = 0.0;
+   ir->IntPoint(np-1).x = 1.0;
+
+   if ( np > 2 )
+   {
+      IntegrationRule gl_ir;
+      GaussLegendre(np-1, &gl_ir);
+
+      for (int i = 1; i < np-1; ++i)
+      {
+         ir->IntPoint(i).x = (gl_ir.IntPoint(i-1).x + gl_ir.IntPoint(i).x)/2;
+      }
+   }
+
+   CalculateUniformWeights(ir, Quadrature1D::ClosedGL);
+}
+
 void QuadratureFunctions1D::GivePolyPoints(const int np, double *pts,
                                            const int type)
 {
@@ -650,6 +670,11 @@ void QuadratureFunctions1D::GivePolyPoints(const int np, double *pts,
          OpenHalfUniform(np, &ir);
          break;
       }
+      case Quadrature1D::ClosedGL:
+      {
+         ClosedGL(np, &ir);
+         break;
+      }
       default:
       {
          MFEM_ABORT("Asking for an unknown type of 1D Quadrature points, "

fem/intrules.hpp

@@ -272,6 +272,7 @@ public:
    void OpenUniform(const int np, IntegrationRule *ir);
    void ClosedUniform(const int np, IntegrationRule *ir);
    void OpenHalfUniform(const int np, IntegrationRule *ir);
+   void ClosedGL(const int np, IntegrationRule *ir);
    ///@}
 
    /// A helper function that will play nice with Poly_1D::OpenPoints and
@@ -293,7 +294,8 @@ public:
       GaussLobatto    = 1,
       OpenUniform     = 2,  ///< aka open Newton-Cotes
       ClosedUniform   = 3,  ///< aka closed Newton-Cotes
-      OpenHalfUniform = 4   ///< aka "open half" Newton-Cotes
+      OpenHalfUniform = 4,  ///< aka "open half" Newton-Cotes
+      ClosedGL        = 5   ///< aka closed Gauss Legendre
    };
    /** @brief If the Quadrature1D type is not closed return Invalid; otherwise
        return type. */

fem/libceed/ceed.cpp

@@ -97,6 +97,8 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
    Vector qweight(Q);
    Vector shape_i(P);
    DenseMatrix grad_i(P, dim);
+
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
    const TensorBasisElement * tfe =
@@ -128,7 +130,15 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
          const int el_offset = fe->GetDof() * i;
          for (int j = 0; j < fe->GetDof(); j++)
          {
-            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+            if (compstride == 1)
+            {
+               tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                          el_dof.GetJ()[dof_map[j] + el_offset];
+            }
+            else
+            {
+               tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+            }
          }
       }
    }
@@ -157,20 +167,23 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
       {
          for (int i = 0; i < P; i++)
          {
-            tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
+            if (compstride == 1)
+            {
+               tp_el_dof[i + e*P] = fes.GetVDim()*el_dof.GetJ()[i + e*P];
+            }
+            else
+            {
+               tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
+            }
          }
       }
    }
    CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
                      fe->GetDof(), ir.GetNPoints(), shape.GetData(),
                      grad.GetData(), qref.GetData(), qweight.GetData(), basis);
-   CeedInterlaceMode imode = CEED_NONINTERLACED;
-   if (fes.GetOrdering()==Ordering::byVDIM)
-   {
-      imode = CEED_INTERLACED;
-   }
-   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
-                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
@@ -215,6 +228,7 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
                            grad1d.GetData(), qref1d.GetData(),
                            qweight1d.GetData(), basis);
 
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
    const Table &el_dof = fes.GetElementToDofTable();
    Array<int> tp_el_dof(el_dof.Size_of_connections());
    for (int i = 0; i < mesh->GetNE(); i++)
@@ -222,16 +236,20 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
       const int el_offset = fe->GetDof() * i;
       for (int j = 0; j < fe->GetDof(); j++)
       {
-         tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+         if (compstride == 1)
+         {
+            tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                       el_dof.GetJ()[dof_map[j] + el_offset];
+         }
+         else
+         {
+            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+         }
       }
    }
-   CeedInterlaceMode imode = CEED_NONINTERLACED;
-   if (fes.GetOrdering()==Ordering::byVDIM)
-   {
-      imode = CEED_INTERLACED;
-   }
-   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
-                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
                              tp_el_dof.GetData(), restr);
 }
 
@@ -298,8 +316,9 @@ void CeedPAAssemble(const CeedPAOperator& op,
    CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
 
    const int qdatasize = op.qdatasize;
-   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, nelem*nqpts, qdatasize,
-                                    CEED_STRIDES_BACKEND, &ceedData.restr_i);
+   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, qdatasize,
+                                    nelem*nqpts*qdatasize, CEED_STRIDES_BACKEND,
+                                    &ceedData.restr_i);
 
    CeedVectorCreate(ceed, mesh->GetNodes()->Size(), &ceedData.node_coords);
    CeedVectorSetArray(ceedData.node_coords, CEED_MEM_HOST, CEED_USE_POINTER,

fem/linearform.cpp

@@ -199,10 +199,19 @@ void LinearForm::Assemble()
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
    fes = f;
-   NewDataAndSize((double *)v + v_offset, fes->GetVSize());
+   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, f->GetVSize()),
+                    f->GetVSize(), false);
    ResetDeltaLocations();
 }
 
+void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
+{
+   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
+   fes = f;
+   v.UseDevice(true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+}
+
 void LinearForm::AssembleDelta()
 {
    if (dlfi_delta.Size() == 0) { return; }

fem/linearform.hpp

@@ -19,14 +19,14 @@
 namespace mfem
 {
 
-/// Class for linear form - Vector with associated FE space and LFIntegrators.
+/// Vector with associated FE space and LinearFormIntegrators.
 class LinearForm : public Vector
 {
 protected:
    /// FE space on which the LinearForm lives. Not owned.
    FiniteElementSpace *fes;
 
-   /** @brief Indicates the LinerFormIntegrator%s stored in #dlfi, #dlfi_delta,
+   /** @brief Indicates the LinearFormIntegrator%s stored in #dlfi, #dlfi_delta,
        #blfi, and #flfi are owned by another LinearForm. */
    int extern_lfs;
 
@@ -175,6 +175,15 @@ public:
        @note This method does not perform assembly. */
    void Update(FiniteElementSpace *f, Vector &v, int v_offset);
 
+   /** @brief Make the LinearForm reference external data on a new
+       FiniteElementSpace. */
+   /** This method changes the FiniteElementSpace associated with the LinearForm
+       @a *f and sets the data of the Vector @a v (plus the @a v_offset)
+       as external data in the LinearForm.
+       @note This version of the method will also perform bounds checks when
+       the build option MFEM_DEBUG is enabled. */
+   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
+
    /// Return the action of the LinearForm as a linear mapping.
    /** Linear forms are linear functionals which map GridFunctions to
        the real numbers.  This method performs this mapping which in

fem/lininteg.cpp

@@ -63,6 +63,53 @@ void DomainLFIntegrator::AssembleDeltaElementVect(
    elvect *= delta->EvalDelta(Trans, Trans.GetIntPoint());
 }
 
+void DomainLFGradIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   int dof = el.GetDof();
+   int spaceDim = Tr.GetSpaceDim();
+
+   dshape.SetSize(dof, spaceDim);
+
+   elvect.SetSize(dof);
+   elvect = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int intorder = 2 * el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), intorder);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetIntPoint(&ip);
+      el.CalcPhysDShape(Tr, dshape);
+
+      Q.Eval(Qvec, Tr, ip);
+      Qvec *= ip.weight * Tr.Weight();
+
+      dshape.AddMult(Qvec, elvect);
+   }
+}
+
+void DomainLFGradIntegrator::AssembleDeltaElementVect(
+   const FiniteElement &fe, ElementTransformation &Trans, Vector &elvect)
+{
+   MFEM_ASSERT(vec_delta != NULL,"coefficient must be VectorDeltaCoefficient");
+   int dof = fe.GetDof();
+   int spaceDim = Trans.GetSpaceDim();
+
+   dshape.SetSize(dof, spaceDim);
+   fe.CalcPhysDShape(Trans, dshape);
+
+   vec_delta->EvalDelta(Qvec, Trans, Trans.GetIntPoint());
+
+   elvect.SetSize(dof);
+   dshape.Mult(Qvec, elvect);
+}
 
 void BoundaryLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
@@ -112,10 +159,13 @@ void BoundaryLFIntegrator::AssembleRHSElementVect(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
 
-      Tr.Face->SetIntPoint (&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+
       double val = Tr.Face->Weight() * ip.weight * Q.Eval(*Tr.Face, ip);
 
       el.CalcShape(eip, shape);
@@ -255,7 +305,6 @@ void VectorDomainLFIntegrator::AssembleDeltaElementVect(
    MultVWt(shape, Qvec, elvec_as_mat);
 }
 
-
 void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -313,10 +362,12 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
       const IntegrationPoint &ip = ir->IntPoint(i);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
 
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
 
       // Use Tr transformation in case Q depends on boundary attribute
       Q.Eval(vec, Tr, ip);
@@ -332,7 +383,6 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
    }
 }
 
-
 void VectorFEDomainLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -362,7 +412,6 @@ void VectorFEDomainLFIntegrator::AssembleRHSElementVect(
 
       QF.Eval (vec, Tr, ip);
       vec *= ip.weight * Tr.Weight();
-
       vshape.AddMult (vec, elvect);
    }
 }
@@ -383,6 +432,125 @@ void VectorFEDomainLFIntegrator::AssembleDeltaElementVect(
    vshape.Mult(vec, elvect);
 }
 
+void VectorFEDomainLFCurlIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   int dof = el.GetDof();
+   int spaceDim = Tr.GetSpaceDim();
+   int n=(spaceDim == 3)? spaceDim : 1;
+   curlshape.SetSize(dof,n);
+   vec.SetSize(n);
+
+   elvect.SetSize(dof);
+   elvect = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int intorder = 2*el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), intorder);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetIntPoint (&ip);
+      el.CalcPhysCurlShape(Tr, curlshape);
+
+      switch (spaceDim)
+      {
+         case 3:
+            MFEM_VERIFY(QF, "VectorFunctionCoefficient not provided");
+            QF->Eval(vec, Tr, ip);
+            break;
+         case 2:
+            MFEM_VERIFY(Q, "FunctionCoefficient (Scalar) not provided");
+            vec[0] = Q->Eval(Tr, ip);
+            break;
+         default:
+            break; // This should be unreachable
+      }
+      vec *= ip.weight * Tr.Weight();
+      curlshape.AddMult (vec, elvect);
+   }
+}
+
+void VectorFEDomainLFCurlIntegrator::AssembleDeltaElementVect(
+   const FiniteElement &fe, ElementTransformation &Trans, Vector &elvect)
+{
+   int spaceDim = Trans.GetSpaceDim();
+   switch (spaceDim)
+   {
+      case 3:
+         MFEM_ASSERT(vec_delta != NULL,
+                     "coefficient must be VectorDeltaCoefficient");
+         break;
+      case 2:
+         MFEM_ASSERT(delta != NULL,
+                     "coefficient must be DeltaCoefficient");
+         break;
+      default:
+         break; // This should be unreachable
+   }
+   int dof = fe.GetDof();
+   int n=(spaceDim == 3)? spaceDim : 1;
+   curlshape.SetSize(dof, n);
+   elvect.SetSize(dof);
+   fe.CalcPhysCurlShape(Trans, curlshape);
+
+   switch (spaceDim)
+   {
+      case 3:
+         vec_delta->EvalDelta(vec, Trans, Trans.GetIntPoint());
+         curlshape.Mult(vec, elvect);
+         break;
+      case 2:
+         curlshape.GetColumn(0,elvect);
+         elvect *= delta->EvalDelta(Trans, Trans.GetIntPoint());
+         break;
+      default:
+         break; // This should be unreachable
+   }
+}
+
+void VectorFEDomainLFDivIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   int dof = el.GetDof();
+
+   divshape.SetSize(dof);       // vector of size dof
+   elvect.SetSize(dof);
+   elvect = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int intorder = 2 * el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), intorder);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetIntPoint (&ip);
+      double val = Tr.Weight() * Q.Eval(Tr, ip);
+      el.CalcPhysDivShape(Tr, divshape);
+
+      add(elvect, ip.weight * val, divshape, elvect);
+   }
+}
+
+void VectorFEDomainLFDivIntegrator::AssembleDeltaElementVect(
+   const FiniteElement &fe, ElementTransformation &Trans, Vector &elvect)
+{
+   MFEM_ASSERT(delta != NULL, "coefficient must be DeltaCoefficient");
+   elvect.SetSize(fe.GetDof());
+   fe.CalcPhysDivShape(Trans, elvect);
+   elvect *= delta->EvalDelta(Trans, Trans.GetIntPoint());
+}
+
 void VectorBoundaryFluxLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -448,7 +616,6 @@ void VectorFEBoundaryFluxLFIntegrator::AssembleRHSElementVect(
    }
 }
 
-
 void VectorFEBoundaryTangentLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -483,7 +650,6 @@ void VectorFEBoundaryTangentLFIntegrator::AssembleRHSElementVect(
    }
 }
 
-
 void BoundaryFlowIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -522,11 +688,13 @@ void BoundaryFlowIntegrator::AssembleRHSElementVect(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
-      el.CalcShape(eip, shape);
 
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+      el.CalcShape(eip, shape);
 
       // Use Tr.Elem1 transformation for u so that it matches the coefficient
       // used with the ConvectionIntegrator and/or the DGTraceIntegrator.
@@ -548,7 +716,6 @@ void BoundaryFlowIntegrator::AssembleRHSElementVect(
    }
 }
 
-
 void DGDirichletLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -592,10 +759,13 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
    for (int p = 0; p < ir->GetNPoints(); p++)
    {
       const IntegrationPoint &ip = ir->IntPoint(p);
-      IntegrationPoint eip;
 
-      Tr.Loc1.Transform(ip, eip);
-      Tr.SetIntPoint(&ip);
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+
       if (dim == 1)
       {
          nor(0) = 2*eip.x - 1.0;
@@ -614,14 +784,14 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
       {
          if (Q)
          {
-            w *= Q->Eval(Tr, ip);
+            w *= Q->Eval(*Tr.Elem1, eip);
          }
          ni.Set(w, nor);
       }
       else
       {
          nh.Set(w, nor);
-         MQ->Eval(mq, Tr, ip);
+         MQ->Eval(mq, *Tr.Elem1, eip);
          mq.MultTranspose(nh, ni);
       }
       CalcAdjugate(Tr.Elem1->Jacobian(), adjJ);
@@ -637,7 +807,6 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
    }
 }
 
-
 void DGElasticityDirichletLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -686,9 +855,12 @@ void DGElasticityDirichletLFIntegrator::AssembleRHSElementVect(
    for (int pi = 0; pi < ir->GetNPoints(); ++pi)
    {
       const IntegrationPoint &ip = ir->IntPoint(pi);
-      IntegrationPoint eip;
-      Tr.Loc1.Transform(ip, eip);
-      Tr.SetIntPoint(&ip);
+
+      // Set the integration point in the face and the neighboring element
+      Tr.SetAllIntPoints(&ip);
+
+      // Access the neighboring element's integration point
+      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
 
       // Evaluate the Dirichlet b.c. using the face transformation.
       uD.Eval(u_dir, Tr, ip);

fem/lininteg.hpp

@@ -119,6 +119,33 @@ public:
    using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
+/// Class for domain integrator L(v) := (f, grad v)
+class DomainLFGradIntegrator : public DeltaLFIntegrator
+{
+private:
+   Vector shape, Qvec;
+   VectorCoefficient &Q;
+   DenseMatrix dshape;
+
+public:
+   /// Constructs the domain integrator (Q, grad v)
+   DomainLFGradIntegrator(VectorCoefficient &QF)
+      : DeltaLFIntegrator(QF), Q(QF) { }
+
+   /** Given a particular Finite Element and a transformation (Tr)
+       computes the element right hand side element vector, elvect. */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect);
+
+   virtual void AssembleDeltaElementVect(const FiniteElement &fe,
+                                         ElementTransformation &Trans,
+                                         Vector &elvect);
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+};
+
+
 /// Class for boundary integration L(v) := (g, v)
 class BoundaryLFIntegrator : public LinearFormIntegrator
 {
@@ -252,6 +279,56 @@ public:
    using LinearFormIntegrator::AssembleRHSElementVect;
 };
 
+/// \f$ (Q, curl v)_{\Omega} \f$ for Nedelec Elements)
+class VectorFEDomainLFCurlIntegrator : public DeltaLFIntegrator
+{
+private:
+   VectorCoefficient *QF=nullptr;
+   Coefficient *Q=nullptr;
+   DenseMatrix curlshape;
+   Vector vec;
+
+public:
+   /// Constructs the domain integrator (Q, curl v)
+   VectorFEDomainLFCurlIntegrator(VectorCoefficient &F)
+      : DeltaLFIntegrator(F), QF(&F) { }
+   VectorFEDomainLFCurlIntegrator(Coefficient &F)
+      : DeltaLFIntegrator(F), Q(&F) { }
+
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect);
+
+   virtual void AssembleDeltaElementVect(const FiniteElement &fe,
+                                         ElementTransformation &Trans,
+                                         Vector &elvect);
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+};
+
+/// \f$ (Q, div v)_{\Omega} \f$ for RT Elements)
+class VectorFEDomainLFDivIntegrator : public DeltaLFIntegrator
+{
+private:
+   Vector divshape;
+   Coefficient &Q;
+public:
+   /// Constructs the domain integrator (Q, div v)
+   VectorFEDomainLFDivIntegrator(Coefficient &QF)
+      : DeltaLFIntegrator(QF), Q(QF) { }
+
+   /** Given a particular Finite Element and a transformation (Tr)
+       computes the element right hand side element vector, elvect. */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect);
+
+   virtual void AssembleDeltaElementVect(const FiniteElement &fe,
+                                         ElementTransformation &Trans,
+                                         Vector &elvect);
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+};
 
 /** \f$ (f, v \cdot n)_{\partial\Omega} \f$ for vector test function
     v=(v1,...,vn) where all vi are in the same scalar FE space and f is a
@@ -283,7 +360,7 @@ class VectorFEBoundaryFluxLFIntegrator : public LinearFormIntegrator
 private:
    Coefficient *F;
    Vector shape;
-   int oa, ob; // these contol the quadrature order, see DomainLFIntegrator
+   int oa, ob; // these control the quadrature order, see DomainLFIntegrator
 
 public:
    VectorFEBoundaryFluxLFIntegrator(int a = 1, int b = -1)