minor changes in sample runs

Change performance miniapps compiler flags for Apple M1

.appveyor.yml

@@ -23,9 +23,8 @@ install:
 - set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
 - set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
 
-# Install METIS, use a mirror because the original source server is not always
-# up. Original url:
-#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
+# Install METIS, use MFEM's mirror because the original source server is often
+# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0

.github/workflows/repo-check.yml

@@ -107,7 +107,7 @@ jobs:
       run: |
         sudo apt-get install doxygen graphviz
         cd doc
-        doxygen -u CodeDocumentation.conf.in 2>/dev/null
+        doxygen -u CodeDocumentation.conf.in
 
     - name: build documentation
       run: |

.gitignore

@@ -275,6 +275,7 @@ miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
+miniapps/tools/check-tmop-metric
 
 miniapps/toys/automata
 miniapps/toys/life
@@ -307,6 +308,7 @@ miniapps/solvers/sol.*
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh
 
+miniapps/multidomain/multidomain
 miniapps/hooke/hooke
 
 # Unit test binary and outputs

CHANGELOG

@@ -15,11 +15,15 @@ Meshing improvements
 --------------------
 - Added support for mixed meshes and pyramids in GSLIB-FindPoints.
 
+- Added new SubMesh and ParSubMesh classes that can be used to extract a subset
+  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
+  and work with all existing MFEM interfaces like finite element spaces etc.
+
 Discretization improvements
 ---------------------------
 - Added support for assembling low-order-refined matrices using a GPU-enabled
   "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration.
+  acceleration with arbitrary user-supplied coefficients.
 
 - Added support for partial assembly and fully matrix-free operators on mixed
   meshes (different element types and p-adaptivity) through libCEED, including
@@ -40,8 +44,28 @@ Discretization improvements
 - Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
   See fem/estimators.hpp.
 
+- Fixes and improvements in LinearFormExtension.
+
+- Added a new class FaceQuadratureSpace that allows for the construction of
+  QuadratureFunctions on the interior or boundary faces of a mesh.
+
+- Added a class CoefficientVector for efficient access of variable coefficient
+  values at quadrature points (in particular for GPU/device kernels).
+
 Linear and nonlinear solvers
 ----------------------------
+- Added a new class DGMassInverse that performs a local elementwise CG
+  iteration to solve systems involving the discontinuous Galerkin mass matrix,
+  including support for device/GPU acceleration.
+
+- Added more flexibility to the constraint solver classes:
+  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
+    (necessary for penalty contact)
+  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
+  * All constraint solver classes can take a user-defined preconditioner
+
+ - Added functions to toggle additional options for the SuperLU_Dist and Hypre
+   preconditioners (ParaSails, Euclid, ILU).
 
 New and updated examples and miniapps
 -------------------------------------
@@ -78,15 +102,23 @@ Miscellaneous
 -------------
 - Various other simplifications, extensions, and bugfixes in the code.
 
-
 - Added boundary elimination with device support for `SparseMatrix` and
   `HypreParMatrix`.
-  
+
 - When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
   outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
   `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).
 
-- Added TMOP metrics for mesh untangling and worst-case quality improvement.
+- Added more 3D TMOP metrics, as well as specialized metrics for mesh
+  untangling and worst-case quality improvement;
+
+- Fully encapsulated SUNDIALS `N_Vector` object within the `SundialsNVector`
+  class by removing deprecated (e.g. `HypreParVector::ToNVector`) and
+  non-deprecated (e.g. `Vector::ToNVector`) functions in other classes.
+
+- The behavior of GridFunction::GetTrueVector() has been changed to not return
+  an empty true vector.
+
 
 Version 4.4, released on March 21, 2022
 =======================================

CMakeLists.txt

@@ -137,7 +137,7 @@ if (MFEM_USE_CUDA)
    set(CUSPARSE_FOUND TRUE)
    set(CUSPARSE_LIBRARIES "cusparse")
    set(CUBLAS_FOUND TRUE)
-   set(CUSBLAS_LIBRARIES "cublas")
+   set(CUBLAS_LIBRARIES "cublas")
 endif()
 
 if (XSDK_ENABLE_C)
@@ -477,12 +477,21 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
   endif()
 endif()
 
+# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
+# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
+#   Target "mfem" links to target "Threads::Threads" but the target was not
+#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
+#   an ALIAS target is missing?
+# Call Stack (most recent call first):
+#   CMakeLists.txt:474 (mfem_add_library)
+find_package(Threads REQUIRED)
+
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist METIS SuiteSparse SUNDIALS
     PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
     ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG

CONTRIBUTING.md

@@ -102,7 +102,9 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   ├── cmake
-  │   └── githooks
+  │   ├── docker
+  │   ├── githooks
+  │   └── vcpkg
   ├── data
   ├── doc
   ├── examples
@@ -111,6 +113,7 @@ The MFEM source code has the following structure:
   │   ├── ginkgo
   │   ├── hiop
   │   ├── jupyter
+  │   ├── moonolith
   │   ├── petsc
   │   ├── pumi
   │   ├── sundials
@@ -118,13 +121,15 @@ The MFEM source code has the following structure:
   ├── fem
   │   ├── ceed
   │   ├── fe
-  │   ├── qinterp
+  │   ├── lor
   │   ├── moonolith
+  │   ├── qinterp
   │   └── tmop
   ├── general
   ├── linalg
   │   └── simd
   ├── mesh
+  │   └── submesh
   ├── miniapps
   │   ├── adjoint
   │   ├── autodiff
@@ -134,6 +139,7 @@ The MFEM source code has the following structure:
   │   ├── hooke
   │   ├── meshing
   │   ├── mtop
+  │   ├── multidomain
   │   ├── navier
   │   ├── nurbs
   │   ├── parelag

INSTALL

@@ -16,7 +16,11 @@ requires an MPI C++ compiler, as well as the following external libraries:
   https://github.com/hypre-space/hypre
 
 - METIS (a family of multilevel partitioning algorithms)
-  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
+  https://github.com/mfem/tpls
+
+  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
+  webpage, https://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
+  and we don't support yet the new repo https://github.com/KarypisLab/METIS.
 
 The hypre dependency can be downloaded as a tarball from GitHub or from the
 project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
@@ -472,10 +476,10 @@ MFEM_USE_CODIPACK = YES/NO
   www.scicomp.uni-kl.de/codi/
 
 MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical 
-  methods and C++ algorithms for working with implicitly-defined geometry and 
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
-  provides interface to Algoim v1. Thus, to check out the specific state use: 
+  Enable the usage of Algoim - a collection of high-order accurate numerical
+  methods and C++ algorithms for working with implicitly-defined geometry and
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM
+  provides interface to Algoim v1. Thus, to check out the specific state use:
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
   https://algoim.github.io
 
@@ -550,7 +554,7 @@ MFEM_USE_FMS = YES/NO
    Enables support for the FMS library which consists of the DataCollection
    sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
    fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   conversion routines between FMS's FmsDataCollection structure and MFEM's
    DataCollection class, see the header file fem/fmsconvert.hpp.
 
 MFEM_USE_PARELAG = YES/NO
@@ -597,7 +601,7 @@ The specific libraries and their options are:
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
+  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
   Options: METIS_OPT, METIS_LIB.
   Versions: METIS 4.0.3 or 5.1.0.
 

config/cmake/modules/FindENZYME.cmake

@@ -19,7 +19,7 @@ if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
   # Set ENZYME_FOUND
   set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
 
-  # Set CXX flags to accomodate the Enzyme Clang plugin
+  # Set CXX flags to accommodate the Enzyme Clang plugin
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
   set(MFEM_USE_ENZYME YES)
 else()

config/cmake/modules/FindHDF5.cmake

@@ -0,0 +1,57 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - HDF5_FOUND - If HDF5 was found
+#   - HDF5_LIBRARIES - The HDF5 libraries
+#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
+
+# First Check for HDF5_DIR
+if(NOT HDF5_DIR)
+    MESSAGE(FATAL_ERROR "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
+endif()
+
+# Find includes
+find_path( HDF5_INCLUDE_DIRS hdf5.h
+           PATHS  ${HDF5_DIR}/include/
+           NO_DEFAULT_PATH
+           NO_CMAKE_ENVIRONMENT_PATH
+           NO_CMAKE_PATH
+           NO_SYSTEM_ENVIRONMENT_PATH
+           NO_CMAKE_SYSTEM_PATH)
+
+find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
+              PATHS ${HDF5_DIR}/lib
+              NO_DEFAULT_PATH
+              NO_CMAKE_ENVIRONMENT_PATH
+              NO_CMAKE_PATH
+              NO_SYSTEM_ENVIRONMENT_PATH
+              NO_CMAKE_SYSTEM_PATH)
+
+find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
+              PATHS ${HDF5_DIR}/lib
+              NO_DEFAULT_PATH
+              NO_CMAKE_ENVIRONMENT_PATH
+              NO_CMAKE_PATH
+              NO_SYSTEM_ENVIRONMENT_PATH
+              NO_CMAKE_SYSTEM_PATH)
+
+set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+
+# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
+# listed variables are TRUE
+find_package_handle_standard_args(HDF5  DEFAULT_MSG
+                                  HDF5_INCLUDE_DIRS
+                                  __HDF5_LIBRARY
+                                  __HDF5_HL_LIBRARY
+                                  HDF5_LIBRARIES )

config/cmake/modules/FindUMPIRE.cmake

@@ -14,6 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS
 
-include(MfemCmakeUtilities)
-mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
-  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")
+find_package(umpire REQUIRED CONFIG)
+set(UMPIRE_FOUND ${umpire_FOUND})
+set(UMPIRE_LIBRARIES "umpire")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -43,22 +43,14 @@ function(convert_filenames_to_full_paths NAMES)
   set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()
 
-# Wrapper for add_executable that calls the HIP wrapper if applicable
+# Wrapper for add_executable
 macro(mfem_add_executable NAME)
-  if (MFEM_USE_HIP)
-    add_executable(${NAME} ${ARGN})
-  else()
-    add_executable(${NAME} ${ARGN})
-  endif()
+  add_executable(${NAME} ${ARGN})
 endmacro()
 
-# Wrapper for add_library that calls the HIP wrapper if applicable
+# Wrapper for add_library
 macro(mfem_add_library NAME)
-  if (MFEM_USE_HIP)
-    add_library(${NAME} ${ARGN})
-  else()
-    add_library(${NAME} ${ARGN})
-  endif()
+  add_library(${NAME} ${ARGN})
 endmacro()
 
 # Simple shortcut to add_custom_target() with option to add the target to the

config/config.hpp

@@ -31,9 +31,11 @@
 
 // Windows specific options
 #ifdef _WIN32
+#ifndef _USE_MATH_DEFINES
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
+#endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
 // posix_memalign(), strdup(), strerror_r().

config/defaults.mk

@@ -179,7 +179,7 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
 endif
 
 # ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
 # the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
 ifeq ($(MFEM_USE_HIP),YES)
    HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
@@ -251,10 +251,11 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
-SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
+ $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
+ -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
  -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
 
 ifeq ($(MFEM_USE_MPI),YES)
@@ -309,7 +310,7 @@ SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortan library:
+# Additional Fortran library:
 # MPI_FORTRAN_LIB += -lgfortran
 
 # MUMPS library configuration

data/tinyzoo-3d.mesh

@@ -1,7 +1,7 @@
 MFEM mesh v1.0
 
 #
-# MFEM Geomety Types (see mesh/geom.hpp):
+# MFEM Geometry Types (see mesh/geom.hpp):
 #
 # POINT       = 0
 # SEGMENT     = 1

examples/amg_tests.cpp

@@ -0,0 +1,267 @@
+//                       MFEM Example 2 - Parallel Version
+//
+
+// mpirun -np 6 ./amg_tests -lambda 10.0 -mu 10.0 -pr 4 -sx 50
+// hypre iterations: 126, 148, 169, 179, 202
+
+// mpirun -np 6 ./amg_tests -lambda 20.0 -mu 10.0 -pr 4 -sx 50 -elast
+// hypre iterations: 100, 94, 123, 171, 355
+
+// mpirun -np 6 ./amg_tests -lambda 2.0 -mu 1.0 -pr 3 -sx 100 -sy 8 -sz 2
+// hypre iterations: 119, 129, 143, 177
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/beam-hex.mesh";
+   int order = 1;
+   bool static_cond = false;
+   bool visualization = 1;
+   double lambda = 1.0;
+   double mu = 1.0;
+   bool amg_elast = 0;
+   int sref = 0;
+   int pref = 0;
+   double sx = 1.0;
+   double sy = 1.0;
+   double sz = 1.0;
+   bool reorder_space = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&sref, "-sr", "--sref",
+                  "Number of serial refinements");
+   args.AddOption(&pref, "-pr", "--pref",
+                  "Number of parallel refinements");
+   args.AddOption(&lambda, "-lambda", "--lambda",
+                  "Lame constant λ");
+   args.AddOption(&mu, "-mu", "--mu",
+                  "Lame constant μ");
+   args.AddOption(&sx, "-sx", "--sx",
+                  "Length in the x direction");
+   args.AddOption(&sy, "-sy", "--sy",
+                  "Length in the y direction");
+   args.AddOption(&sz, "-sz", "--sz",
+                  "Length in the z direction");
+   args.AddOption(&amg_elast, "-elast", "--amg-for-elasticity", "-sys",
+                  "--amg-for-systems",
+                  "Use the special AMG elasticity solver (GM/LN approaches), "
+                  "or standard AMG for systems (unknown approach).");
+   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
+                  "Use byNODES ordering of vector space instead of byVDIM");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   Mesh mesh = Mesh::MakeCartesian3D((int)sx, (int)sy, int(sz),
+                                     mfem::Element::HEXAHEDRON,
+                                     sx,sy,sz);
+
+   int dim = mesh.Dimension();
+
+   //set attributes
+   for (int i = 0; i<mesh.GetNBE(); i++)
+   {
+      Element * be = mesh.GetBdrElement(i);
+      Array<int> vertices;
+      be->GetVertices(vertices);
+
+      double * coords0 = mesh.GetVertex(vertices[0]);
+      double * coords1 = mesh.GetVertex(vertices[1]);
+      double * coords2 = mesh.GetVertex(vertices[2]);
+      double * coords3 = mesh.GetVertex(vertices[3]);
+
+
+
+      Vector center(3);
+      center(0) = 0.25*(coords0[0] + coords1[0] + coords2[0] + coords3[0]);
+
+      if (abs(center(0) - 0.0) < 1e-10)
+      {
+         // the left face
+         be->SetAttribute(1);
+      }
+      else if (abs(center(0) - sx) < 1e-10)
+      {
+         // the right face
+         be->SetAttribute(2);
+      }
+      else
+      {
+         // all other boundaries
+         be->SetAttribute(3);
+      }
+   }
+   mesh.SetAttributes();
+
+
+   for (int l = 0; l < sref; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace;
+   if (reorder_space)
+   {
+      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
+   }
+   else
+   {
+      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
+   }
+
+   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl
+           << "Assembling: " << flush;
+   }
+
+   Array<int> ess_tdof_list, ess_bdr(pmesh->bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+
+   VectorArrayCoefficient f(dim);
+   for (int i = 0; i < dim-1; i++)
+   {
+      f.Set(i, new ConstantCoefficient(0.0));
+   }
+   {
+      Vector pull_force_z(pmesh->bdr_attributes.Max());
+      pull_force_z = 0.0;
+      pull_force_z(1) = -1.0e-4;
+      f.Set(dim-1, new PWConstCoefficient(pull_force_z));
+      Vector pull_force_y(pmesh->bdr_attributes.Max());
+      pull_force_y = 0.0;
+      pull_force_y(1) = -1.0e-3;
+      f.Set(dim-2, new PWConstCoefficient(pull_force_y));
+   }
+
+   ParLinearForm *b = new ParLinearForm(fespace);
+   b->AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
+
+   ParGridFunction x(fespace);
+   x = 0.0;
+
+   ConstantCoefficient lambda_cf(lambda);
+   ConstantCoefficient mu_cf(mu);
+
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_cf, mu_cf));
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   for (int i = 0; i<=pref; i++)
+   {
+      a->Assemble();
+      b->Assemble();
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      HypreParMatrix A;
+      Vector B, X;
+      a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+      if (myid == 0)
+      {
+         cout << "done." << endl;
+         cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
+      }
+
+      HypreBoomerAMG *amg = new HypreBoomerAMG(A);
+      if (amg_elast && !a->StaticCondensationIsEnabled())
+      {
+         amg->SetElasticityOptions(fespace);
+      }
+      else
+      {
+         amg->SetSystemsOptions(dim, reorder_space);
+      }
+
+      amg->SetPrintLevel(0);
+      CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetRelTol(1e-8);
+      pcg->SetMaxIter(500);
+      pcg->SetPrintLevel(3);
+      pcg->SetPreconditioner(*amg);
+      pcg->SetOperator(A);
+      pcg->Mult(B, X);
+
+      // 15. Recover the parallel grid function corresponding to X. This is the
+      //     local finite element solution on each processor.
+      a->RecoverFEMSolution(X, *b, x);
+
+      pmesh->SetNodalFESpace(fespace);
+
+      GridFunction *nodes = pmesh->GetNodes();
+      *nodes += x;
+      x *= -1;
+
+      if (visualization)
+      {
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream sol_sock(vishost, visport);
+         sol_sock << "parallel " << num_procs << " " << myid << "\n";
+         sol_sock.precision(8);
+         sol_sock << "solution\n" << *pmesh << x << flush;
+      }
+      *nodes += x;
+
+      // 19. Free the used memory.
+      delete pcg;
+      delete amg;
+
+      if (i == pref)
+      {
+         break;
+      }
+
+      pmesh->UniformRefinement();
+      fespace->Update();
+      a->Update();
+      b->Update();
+      x.Update();
+   }
+
+   delete a;
+   delete b;
+   delete fespace;
+   delete fec;
+   delete pmesh;
+
+   return 0;
+}

examples/ex27p.cpp

@@ -195,7 +195,7 @@ int main(int argc, char *argv[])
    Array<int> ess_tdof_list(0);
    if (h1 && pmesh.bdr_attributes.Size())
    {
-      // For a continuous basis the linear system must be modifed to enforce an
+      // For a continuous basis the linear system must be modified to enforce an
       // essential (Dirichlet) boundary condition. In the DG case this is not
       // necessary as the boundary condition will only be enforced weakly.
       fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);

examples/petsc/ex9p.cpp

@@ -450,7 +450,7 @@ int main(int argc, char *argv[])
       for (int ti = 0; !done; )
       {
          // We cannot match exactly the time history of the Run method
-         // since we are explictly telling PETSc to use a time step
+         // since we are explicitly telling PETSc to use a time step
          double dt_real = min(dt, t_final - t);
          ode_solver->Step(*U, t, dt_real);
          ti++;

examples/sundials/ex10.cpp

@@ -210,6 +210,9 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
 
 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
    // 1. Parse command-line options.
    const char *mesh_file = "../../data/beam-quad.mesh";
    int ref_levels = 2;

examples/sundials/ex10p.cpp

@@ -215,10 +215,11 @@ void visualize(ostream &os, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
    Mpi::Init(argc, argv);
    int myid = Mpi::WorldRank();
    Hypre::Init();
+   Sundials::Init();
 
    // 2. Parse command-line options.
    const char *mesh_file = "../../data/beam-quad.mesh";

examples/sundials/ex16.cpp

@@ -109,6 +109,9 @@ double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
    // 1. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
    int ref_levels = 2;
@@ -290,7 +293,10 @@ int main(int argc, char *argv[])
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 11)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
          ode_solver = arkode; break;
       case 12:
          arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);

examples/sundials/ex16p.cpp

@@ -101,11 +101,12 @@ double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
    Mpi::Init(argc, argv);
    int num_procs = Mpi::WorldSize();
    int myid = Mpi::WorldRank();
    Hypre::Init();
+   Sundials::Init();
 
    // 2. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
@@ -327,7 +328,10 @@ int main(int argc, char *argv[])
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 11)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
          ode_solver = arkode; break;
       case 12:
          arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);

examples/sundials/ex9.cpp

@@ -140,6 +140,9 @@ public:
 
 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
    // 1. Parse command-line options.
    problem = 0;
    const char *mesh_file = "../../data/periodic-hexagon.mesh";
@@ -408,7 +411,7 @@ int main(int argc, char *argv[])
          arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(FEHLBERG_13_7_8);
+         arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
          ode_solver = arkode; break;
    }
 

examples/sundials/ex9p.cpp

@@ -152,11 +152,12 @@ public:
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
    Mpi::Init(argc, argv);
    int num_procs = Mpi::WorldSize();
    int myid = Mpi::WorldRank();
    Hypre::Init();
+   Sundials::Init();
 
    // 2. Parse command-line options.
    problem = 0;
@@ -487,7 +488,10 @@ int main(int argc, char *argv[])
          arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 9)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
          ode_solver = arkode; break;
    }
 

fem/CMakeLists.txt

@@ -39,6 +39,7 @@ set(SRCS
   complex_fem.cpp
   convergence.cpp
   datacollection.cpp
+  dgmassinv.cpp
   doftrans.cpp
   eltrans.cpp
   estimators.cpp
@@ -72,6 +73,7 @@ set(SRCS
   linearform.cpp
   linearform_ext.cpp
   lininteg.cpp
+  lininteg_boundary.cpp
   lininteg_domain.cpp
   lininteg_domain_grad.cpp
   lor/lor.cpp
@@ -88,6 +90,7 @@ set(SRCS
   fespacehierarchy.cpp
   nonlininteg_vectorconvection.cpp
   nonlininteg_vectorconvection_mf.cpp
+  qfunction.cpp
   qinterp/det.cpp
   qinterp/eval_by_nodes.cpp
   qinterp/eval_by_vdim.cpp
@@ -95,6 +98,7 @@ set(SRCS
   qinterp/grad_by_vdim.cpp
   qinterp/grad_phys_by_nodes.cpp
   qinterp/grad_phys_by_vdim.cpp
+  qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
   restriction.cpp
@@ -136,10 +140,13 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
+  bilininteg_mass_pa.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
   datacollection.hpp
+  dgmassinv.hpp
+  dgmassinv_kernels.hpp
   doftrans.hpp
   eltrans.hpp
   estimators.hpp
@@ -189,9 +196,11 @@ set(HDRS
   nonlinearform.hpp
   nonlinearform_ext.hpp
   nonlininteg.hpp
+  qfunction.hpp
   qinterp/dispatch.hpp
   qinterp/eval.hpp
   qinterp/grad.hpp
+  qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
   restriction.hpp

fem/bilinearform.cpp

@@ -992,6 +992,7 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs_,
       mat_e = new SparseMatrix(height);
    }
 
+   vdofs_.HostRead();
    for (int i = 0; i < vdofs_.Size(); i++)
    {
       int vdof = vdofs_[i];

fem/bilinearform.hpp

@@ -333,7 +333,7 @@ public:
 
 
    /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
+        to it.  Used for transferring ownership. */
    SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }
 
    /** @brief Returns a const reference to the sparse matrix of eliminated b.c.:
@@ -774,7 +774,7 @@ public:
    SparseMatrix &SpMat() { return *mat; }
 
    /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
+        to it.  Used for transferring ownership. */
    SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }
 
    /// Adds a domain integrator. Assumes ownership of @a bfi.

fem/bilinearform_ext.cpp

@@ -160,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultMF(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
       }
    }
 
@@ -176,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultMF(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
    }
 }
@@ -217,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultTransposeMF(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
       }
    }
 
@@ -233,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultTransposeMF(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
    }
 }
@@ -418,7 +418,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
       }
    }
 
@@ -434,7 +434,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultPA(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
    }
 }
@@ -475,7 +475,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultTransposePA(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
       }
    }
 
@@ -491,7 +491,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultTransposePA(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
    }
 }
@@ -668,7 +668,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             Y(j, 0, f) += res;
          });
          // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
       }
    }
 
@@ -699,7 +699,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             Y(j, f) += res;
          });
          // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
    }
 }
@@ -796,7 +796,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             Y(j, 0, f) += res;
          });
          // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
       }
    }
 
@@ -827,7 +827,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             Y(j, f) += res;
          });
          // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
    }
 }
@@ -975,6 +975,9 @@ void FABilinearFormExtension::RAP(OperatorHandle &A)
 void FABilinearFormExtension::EliminateBC(const Array<int> &ess_dofs,
                                           OperatorHandle &A)
 {
+   MFEM_VERIFY(a->diag_policy == DiagonalPolicy::DIAG_ONE,
+               "Only DiagonalPolicy::DIAG_ONE supported with"
+               " FABilinearFormExtension.");
 #ifdef MFEM_USE_MPI
    if ( dynamic_cast<ParBilinearForm*>(a) )
    {

fem/bilininteg.cpp

@@ -2003,6 +2003,83 @@ void CurlCurlIntegrator::AssembleElementMatrix
    }
 }
 
+void CurlCurlIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                                const FiniteElement &test_fe,
+                                                ElementTransformation &Trans,
+                                                DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   dim = trial_fe.GetDim();
+   int dimc = trial_fe.GetCurlDim();
+   double w;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector D;
+   DenseMatrix curlshape(tr_nd,dimc), curlshape_dFt(tr_nd,dimc), M;
+   DenseMatrix te_curlshape(te_nd,dimc), te_curlshape_dFt(te_nd,dimc);
+#else
+   curlshape.SetSize(tr_nd,dimc);
+   curlshape_dFt.SetSize(tr_nd,dimc);
+   te_curlshape.SetSize(te_nd,dimc);
+   te_curlshape_dFt.SetSize(te_nd,dimc);
+#endif
+   elmat.SetSize(te_nd, tr_nd);
+
+   if (MQ) { M.SetSize(dimc); }
+   if (DQ) { D.SetSize(dimc); }
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (trial_fe.Space() == FunctionSpace::Pk)
+      {
+         order = test_fe.GetOrder() + trial_fe.GetOrder() - 2;
+      }
+      else
+      {
+         order = test_fe.GetOrder() + trial_fe.GetOrder() + trial_fe.GetDim() - 1;
+      }
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Trans.SetIntPoint(&ip);
+
+      w = ip.weight * Trans.Weight();
+      trial_fe.CalcPhysCurlShape(Trans, curlshape_dFt);
+      test_fe.CalcPhysCurlShape(Trans, te_curlshape_dFt);
+
+      if (MQ)
+      {
+         MQ->Eval(M, Trans, ip);
+         M *= w;
+         Mult(te_curlshape_dFt, M, te_curlshape);
+         AddMultABt(te_curlshape, curlshape_dFt, elmat);
+      }
+      else if (DQ)
+      {
+         DQ->Eval(D, Trans, ip);
+         D *= w;
+         AddMultADBt(te_curlshape_dFt,D,curlshape_dFt,elmat);
+      }
+      else
+      {
+         if (Q)
+         {
+            w *= Q->Eval(Trans, ip);
+         }
+         curlshape_dFt *= w;
+         AddMultABt(te_curlshape_dFt, curlshape_dFt, elmat);
+      }
+   }
+}
+
 void CurlCurlIntegrator
 ::ComputeElementFlux(const FiniteElement &el, ElementTransformation &Trans,
                      Vector &u, const FiniteElement &fluxelem, Vector &flux,
@@ -2240,6 +2317,84 @@ double VectorCurlCurlIntegrator::GetElementEnergy(
    return 0.5 * energy;
 }
 
+void MixedCurlIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans, DenseMatrix &elmat)
+{
+   int dim = trial_fe.GetDim();
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dimc = (dim == 3) ? 3 : 1;
+
+   MFEM_VERIFY(trial_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
+               (dim == 2 && trial_fe.GetMapType() == mfem::FiniteElement::VALUE),
+               "Trial finite element must be either 2D/3D H(Curl) or 2D H1");
+   MFEM_VERIFY(test_fe.GetMapType() == mfem::FiniteElement::VALUE ||
+               test_fe.GetMapType() == mfem::FiniteElement::INTEGRAL,
+               "Test finite element must be in H1/L2");
+
+   bool spaceH1 = (trial_fe.GetMapType() == mfem::FiniteElement::VALUE);
+
+   if (spaceH1)
+   {
+      dshape.SetSize(trial_dof,dim);
+      curlshape.SetSize(dim*trial_dof,1);
+      dimc = dim;
+   }
+   else
+   {
+      curlshape.SetSize(trial_dof,dimc);
+      elmat_comp.SetSize(test_dof, trial_dof);
+   }
+   elmat.SetSize(dimc * test_dof, trial_dof);
+   shape.SetSize(test_dof);
+   elmat = 0.0;
+
+   double c;
+   Vector d_col;
+   const IntegrationRule *ir = IntRule;
+
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderJ();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+      if (spaceH1)
+      {
+         trial_fe.CalcPhysDShape(Trans, dshape);
+         dshape.GradToCurl(curlshape);
+      }
+      else
+      {
+         trial_fe.CalcPhysCurlShape(Trans, curlshape);
+      }
+      test_fe.CalcPhysShape(Trans, shape);
+      c = ip.weight*Trans.Weight();
+      if (Q)
+      {
+         c *= Q->Eval(Trans, ip);
+      }
+      shape *= c;
+
+      for (int d = 0; d < dimc; ++d)
+      {
+         double * curldata = &(curlshape.GetData())[d*trial_dof];
+         for (int jj = 0; jj < trial_dof; ++jj)
+         {
+            for (int ii = 0; ii < test_dof; ++ii)
+            {
+               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+            }
+         }
+      }
+   }
+}
+
 
 void VectorFEMassIntegrator::AssembleElementMatrix(
    const FiniteElement &el,
@@ -2586,6 +2741,54 @@ void DivDivIntegrator::AssembleElementMatrix(
    }
 }
 
+void DivDivIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   double c;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector divshape(tr_nd);
+   Vector te_divshape(te_nd);
+#else
+   divshape.SetSize(tr_nd);
+   te_divshape.SetSize(te_nd);
+#endif
+   elmat.SetSize(te_nd,tr_nd);
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(test_fe.GetOrder(),
+                          trial_fe.GetOrder()) - 2; // <--- OK for RTk
+      ir = &IntRules.Get(test_fe.GetGeomType(), order);
+   }
+
+   elmat = 0.0;
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      trial_fe.CalcDivShape(ip,divshape);
+      test_fe.CalcDivShape(ip,te_divshape);
+
+      Trans.SetIntPoint (&ip);
+      c = ip.weight / Trans.Weight();
+
+      if (Q)
+      {
+         c *= Q -> Eval (Trans, ip);
+      }
+
+      te_divshape *= c;
+      AddMultVWt(te_divshape, divshape, elmat);
+   }
+}
 
 void VectorDiffusionIntegrator::AssembleElementMatrix(
    const FiniteElement &el,
@@ -3780,7 +3983,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
       for (i = 0; i < ndof1; i++)
          for (j = 0; j < face_ndof; j++)
          {
-            elmat(i, j) -= shape1_n(i) * face_shape(j);
+            elmat(i, j) += shape1_n(i) * face_shape(j);
          }
       if (ndof2)
       {
@@ -3788,7 +3991,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
          for (i = 0; i < ndof2; i++)
             for (j = 0; j < face_ndof; j++)
             {
-               elmat(ndof1+i, j) += shape2_n(i) * face_shape(j);
+               elmat(ndof1+i, j) -= shape2_n(i) * face_shape(j);
             }
       }
    }

fem/bilininteg.hpp

@@ -2174,6 +2174,7 @@ public:
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
 class MassIntegrator: public BilinearFormIntegrator
 {
+   friend class DGMassInverse;
 protected:
 #ifndef MFEM_THREAD_SAFE
    Vector shape, te_shape;
@@ -2524,6 +2525,7 @@ private:
 #ifndef MFEM_THREAD_SAFE
    Vector D;
    DenseMatrix curlshape, curlshape_dFt, M;
+   DenseMatrix te_curlshape, te_curlshape_dFt;
    DenseMatrix vshape, projcurl;
 #endif
 
@@ -2557,6 +2559,11 @@ public:
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
 
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
    virtual void ComputeElementFlux(const FiniteElement &el,
                                    ElementTransformation &Trans,
                                    Vector &u, const FiniteElement &fluxelem,
@@ -2602,6 +2609,35 @@ public:
                                    const Vector &elfun);
 };
 
+/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is
+    an optional scalar coefficient, and v is a vector with components v_i in
+    the L2 or H1 space. This integrator handles 3 cases:
+    (a) u ∈ H(curl) in 3D, v is a 3D vector with components v_i in L^2 or H^1
+    (b) u ∈ H(curl) in 2D, v is a scalar field in L^2 or H^1
+    (c) u is a scalar field in H^1, i.e, curl u := [0 1;-1 0]grad u and v is a
+        2D vector field with components v_i in L^2 or H^1 space.
+    Note: Case (b) can also be handled by MixedScalarCurlIntegrator  */
+class MixedCurlIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+
+private:
+   Vector shape;
+   DenseMatrix dshape;
+   DenseMatrix curlshape;
+   DenseMatrix elmat_comp;
+public:
+   MixedCurlIntegrator() : Q{NULL} { }
+   MixedCurlIntegrator(Coefficient *q_) :  Q{q_} { }
+   MixedCurlIntegrator(Coefficient &q) :  Q{&q} { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+};
+
 /** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
     vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
     H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
@@ -2725,7 +2761,7 @@ protected:
 
 private:
 #ifndef MFEM_THREAD_SAFE
-   Vector divshape;
+   Vector divshape, te_divshape;
 #endif
 
    // PA extension
@@ -2743,6 +2779,12 @@ public:
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
    const Coefficient *GetCoefficient() const { return Q; }
 };
 

fem/bilininteg_convection_pa.cpp

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/convection/convection.hpp"
 #include "quadinterpolator.hpp"
 
@@ -1408,66 +1409,10 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * ne, mt);
-   Vector vel;
-   if (VectorConstantCoefficient *cQ =
-          dynamic_cast<VectorConstantCoefficient*>(Q))
-   {
-      vel = cQ->GetVec();
-   }
-   else if (VectorGridFunctionCoefficient *vgfQ =
-               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
-   {
-      vel.SetSize(dim * nq * ne, mt);
 
-      const GridFunction *gf = vgfQ->GetGridFunction();
-      const FiniteElementSpace &gf_fes = *gf->FESpace();
-      const QuadratureInterpolator *qi(gf_fes.GetQuadratureInterpolator(*ir));
-      const bool use_tensor_products = UsesTensorBasis(gf_fes);
-      const ElementDofOrdering ordering = use_tensor_products ?
-                                          ElementDofOrdering::LEXICOGRAPHIC :
-                                          ElementDofOrdering::NATIVE;
-      const Operator *R = gf_fes.GetElementRestriction(ordering);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector vel(*Q, qs, CoefficientStorage::COMPRESSED);
 
-      Vector xe(R->Height(), mt);
-      xe.UseDevice(true);
-
-      R->Mult(*gf, xe);
-      qi->SetOutputLayout(QVectorLayout::byVDIM);
-      qi->DisableTensorProducts(!use_tensor_products);
-      qi->Values(xe,vel);
-   }
-   else if (VectorQuadratureFunctionCoefficient* vqfQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      vel.SetSize(dim * nq * ne);
-      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      DenseMatrix MQ_ir;
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         Q->Eval(MQ_ir, T, *ir);
-         for (int q = 0; q < nq; ++q)
-         {
-            for (int i = 0; i < dim; ++i)
-            {
-               C(i,q,e) = MQ_ir(i,q);
-            }
-         }
-      }
-   }
    PAConvectionSetup(dim, nq, ne, ir->GetWeights(), geom->J,
                      vel, alpha, pa_data);
 }

fem/bilininteg_dgtrace_pa.cpp

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "restriction.hpp"
 
 using namespace std;
@@ -161,88 +162,24 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
-   Vector vel;
-   if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
-                                        (u))
-   {
-      vel = c_u->GetVec();
-   }
-   else if (VectorQuadratureFunctionCoefficient* qf_u =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
-   {
-      // Assumed to be in lexicographical ordering
-      const QuadratureFunction &qFun = qf_u->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
 
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
+   FaceQuadratureSpace qs(*mesh, *ir, type);
+   CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED);
+
+   CoefficientVector r(qs, CoefficientStorage::COMPRESSED);
+   if (rho == nullptr)
    {
-      vel.SetSize(dim * nq * nf);
-      auto C = Reshape(vel.HostWrite(), dim, nq, nf);
-      Vector Vq(dim);
-      int f_ind = 0;
-      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
-      {
-         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse())
-         {
-            // We skip nonconforming coarse faces as they are treated
-            // by the corresponding nonconforming fine faces.
-            continue;
-         }
-         else if ( face.IsOfFaceType(type) )
-         {
-            const int mask = FaceElementTransformations::HAVE_ELEM1 |
-                             FaceElementTransformations::HAVE_LOC1;
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f, mask);
-            for (int q = 0; q < nq; ++q)
-            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                      quad1D, q);
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               u->Eval(Vq, *T.Elem1, eip1);
-               for (int i = 0; i < dim; ++i)
-               {
-                  C(i,iq,f_ind) = Vq(i);
-               }
-            }
-            f_ind++;
-         }
-      }
-      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+      r.SetConstant(1.0);
    }
-   Vector r;
-   if (rho==nullptr)
+   else if (ConstantCoefficient *const_rho = dynamic_cast<ConstantCoefficient*>
+                                             (rho))
    {
-      r.SetSize(1);
-      r(0) = 1.0;
-   }
-   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
-   {
-      r.SetSize(1);
-      r(0) = c_rho->constant;
+      r.SetConstant(const_rho->constant);
    }
    else if (QuadratureFunctionCoefficient* qf_rho =
                dynamic_cast<QuadratureFunctionCoefficient*>(rho))
    {
-      const QuadratureFunction &qFun = qf_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+      r.MakeRef(qf_rho->GetQuadFunction());
    }
    else
    {
@@ -254,45 +191,42 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
       {
          Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse())
+         if (face.IsNonconformingCoarse() || !face.IsOfFaceType(type))
          {
             // We skip nonconforming coarse faces as they are treated
             // by the corresponding nonconforming fine faces.
             continue;
          }
-         else if ( face.IsOfFaceType(type) )
+         FaceElementTransformations &T =
+            *fes.GetMesh()->GetFaceElementTransformations(f);
+         for (int q = 0; q < nq; ++q)
          {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
-            for (int q = 0; q < nq; ++q)
+            // Convert to lexicographic ordering
+            int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                   quad1D, q);
+
+            T.SetAllIntPoints(&ir->IntPoint(q));
+            const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+            const IntegrationPoint &eip2 = T.GetElement2IntPoint();
+            double rq;
+
+            if (face.IsBoundary())
             {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                      quad1D, q);
-
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-               double rq;
-
-               if ( face.IsBoundary() )
-               {
-                  rq = rho->Eval(*T.Elem1, eip1);
-               }
-               else
-               {
-                  double udotn = 0.0;
-                  for (int d=0; d<dim; ++d)
-                  {
-                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
-                  }
-                  if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
-                  else { rq = rho->Eval(*T.Elem1, eip1); }
-               }
-               C(iq,f_ind) = rq;
+               rq = rho->Eval(*T.Elem1, eip1);
             }
-            f_ind++;
+            else
+            {
+               double udotn = 0.0;
+               for (int d=0; d<dim; ++d)
+               {
+                  udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+               }
+               if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
+               else { rq = rho->Eval(*T.Elem1, eip1); }
+            }
+            C(iq,f_ind) = rq;
          }
+         f_ind++;
       }
       MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
    }

fem/bilininteg_diffusion_pa.cpp

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"
 
 using namespace std;
@@ -390,120 +391,21 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
-   int coeffDim = 1;
-   Vector coeff;
-   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
-   if (auto *SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ))
-   {
-      MFEM_VERIFY(SMQ->GetSize() == dim, "");
-      coeffDim = symmDims;
-      coeff.SetSize(symmDims * nq * ne);
 
-      DenseSymmetricMatrix sym_mat;
-      sym_mat.SetSize(dim);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);
 
-      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);
+   if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (VQ) { coeff.Project(*VQ); }
+   else if (Q) { coeff.Project(*Q); }
+   else { coeff.SetConstant(1.0); }
 
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            SMQ->Eval(sym_mat, *tr, ir->IntPoint(p));
-            int cnt = 0;
-            for (int i=0; i<dim; ++i)
-               for (int j=i; j<dim; ++j, ++cnt)
-               {
-                  C(cnt, p, e) = sym_mat(i,j);
-               }
-         }
-      }
-   }
-   else if (MQ)
-   {
-      symmetric = false;
-      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dims*dims);
+   const int pa_size = symmetric ? symmDims : dims*dims;
 
-      coeffDim = MQfullDim;
-
-      coeff.SetSize(MQfullDim * nq * ne);
-
-      DenseMatrix mat;
-      mat.SetSize(dim);
-
-      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            MQ->Eval(mat, *tr, ir->IntPoint(p));
-            for (int i=0; i<dim; ++i)
-               for (int j=0; j<dim; ++j)
-               {
-                  C(j+(i*dim), p, e) = mat(i,j);
-               }
-         }
-      }
-   }
-   else if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "");
-      coeffDim = VQ->GetVDim();
-      coeff.SetSize(coeffDim * nq * ne);
-      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-      Vector DM(coeffDim);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            VQ->Eval(DM, *tr, ir->IntPoint(p));
-            for (int i=0; i<coeffDim; ++i)
-            {
-               C(i, p, e) = DM[i];
-            }
-         }
-      }
-   }
-   else if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
+   pa_data.SetSize(pa_size * nq * ne, mt);
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeff_dim, ne, ir->GetWeights(),
                     geom->J, coeff, pa_data);
 }
 

fem/bilininteg_gradient.cpp

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 
 using namespace std;
 
@@ -209,44 +210,8 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
                "PA requires test and trial space to have same number of quadrature points!");
    pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());
 
-   Vector coeff;
-
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *trial_fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
 
    PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
                    ne, ir->GetWeights(), geom->J, coeff, pa_data);
@@ -865,4 +830,3 @@ void GradientIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 }
 
 } // namespace mfem
-

fem/bilininteg_hcurl.cpp

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qspace.hpp"
 
 using namespace std;
 
@@ -967,8 +968,6 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
-   const int dimc = (dim == 3) ? 3 : 1;
-
    ne = fes.GetNE();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -978,88 +977,19 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
+   if (Q) { coeff.Project(*Q); }
+   else if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
 
-   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
-   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
-
-   symmetric = (SMQ || MQ == NULL);
-
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dim*dim);
+   const int sym_dims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int ndata = (dim == 2) ? 1 : (symmetric ? sym_dims : dim*dim);
    pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
 
-   Vector coeff(coeffDim * ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ)
-   {
-      Vector DM(DQ ? coeffDim : 0);
-      DenseMatrix GM;
-      DenseSymmetricMatrix SM;
-
-      if (DQ)
-      {
-         MFEM_VERIFY(coeffDim == dimc, "");
-      }
-      if (SMQ)
-      {
-         SM.SetSize(dimc);
-         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
-      }
-      else if (MQ)
-      {
-         GM.SetSize(dimc);
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            if (SMQ)
-            {
-               SMQ->Eval(SM, *tr, ir->IntPoint(p));
-
-               int cnt = 0;
-               for (int i=0; i<dimc; ++i)
-                  for (int j=i; j<dimc; ++j, ++cnt)
-                  {
-                     coeffh(cnt, p, e) = SM(i,j);
-                  }
-
-            }
-            else if (MQ)
-            {
-               MQ->Eval(GM, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dimc; ++i)
-                  for (int j=0; j<dimc; ++j)
-                  {
-                     coeffh(j+(i*dimc), p, e) = GM(i,j);
-                  }
-
-            }
-            else if (DQ)
-            {
-               DQ->Eval(DM, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = DM[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
-
    if (el->GetDerivType() != mfem::FiniteElement::CURL)
    {
       MFEM_ABORT("Unknown kernel.");
@@ -1067,7 +997,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    if (dim == 3)
    {
-      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
+      PACurlCurlSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J, coeff,
                         pa_data);
    }
    else
@@ -3489,20 +3419,8 @@ void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    pa_data.SetSize(nq * ne, Device::GetMemoryType());
 
-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), nq, ne);
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeffh(p, e) = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
 
    if (dim == 2)
    {
@@ -3593,38 +3511,11 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
    const int ndata = curlSpaces ? (coeffDim == 1 ? 1 : 9) : symmDims;
    pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
 
-   Vector coeff(coeffDim * nq * ne);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ)
-   {
-      Vector V(coeffDim);
-      if (DQ)
-      {
-         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-
-         for (int p=0; p<nq; ++p)
-         {
-            if (DQ)
-            {
-               DQ->Eval(V, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = V[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
 
    if (testType == mfem::FiniteElement::CURL &&
        trialType == mfem::FiniteElement::CURL && dim == 3)
@@ -5146,38 +5037,11 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
 
    pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
 
-   Vector coeff(coeffDim * nq * ne);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ)
-   {
-      Vector V(coeffDim);
-      if (DQ)
-      {
-         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-
-         for (int p=0; p<nq; ++p)
-         {
-            if (DQ)
-            {
-               DQ->Eval(V, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = V[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
 
    if (trialType == mfem::FiniteElement::CURL && dim == 3)
    {

fem/bilininteg_hdiv.cpp

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qspace.hpp"
 
 using namespace std;
 
@@ -1513,19 +1514,8 @@ void DivDivIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    pa_data.SetSize(nq * ne, Device::GetMemoryType());
 
-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
 
    if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
    {
@@ -1783,19 +1773,8 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    pa_data.SetSize(nq * ne, Device::GetMemoryType());
 
-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
 
    if (test_el->GetMapType() == FiniteElement::INTEGRAL)
    {

fem/bilininteg_mass_pa.cpp

@@ -12,7 +12,9 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/mass/mass.hpp"
+#include "bilininteg_mass_pa.hpp"
 
 using namespace std;
 
@@ -60,43 +62,10 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(ne*nq, mt);
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
 
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
    if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
    if (dim==2)
    {
@@ -590,85 +559,18 @@ static void PAMassApply2D(const int NE,
                           const int d1d = 0,
                           const int q1d = 0)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b_.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto X = Reshape(x_.Read(), D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
    MFEM_FORALL(e, NE,
    {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         double sol_x[max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            sol_x[qy] = 0.0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double s = X(dx,dy,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] += B(qx,dx)* s;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double d2q = B(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] += d2q * sol_x[qx];
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] *= D(qx,qy,e);
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double sol_x[max_D1D];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] = 0.0;
-         }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double s = sol_xy[qy][qx];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] += Bt(dx,qx) * s;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            const double q2d = Bt(dy,qy);
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               Y(dx,dy,e) += q2d * sol_x[dx];
-            }
-         }
-      }
+      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
    });
 }
 
@@ -690,108 +592,13 @@ static void SmemPAMassApply2D(const int NE,
    constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
    MFEM_VERIFY(D1D <= MD1, "");
    MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   const auto b = b_.Read();
+   const auto D = d_.Read();
+   const auto x = x_.Read();
+   auto Y = y_.ReadWrite();
    MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
    {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double BBt[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) BBt;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
-      MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
-      double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
-      double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
-      double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
-      double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            X[dy][dx] = x(dx,dy,e);
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][dy] = b(q,dy);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double dq = 0.0;
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               dq += X[dy][dx] * B[qx][dx];
-            }
-            DQ[dy][qx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double qq = 0.0;
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               qq += DQ[dy][qx] * B[qy][dy];
-            }
-            QQ[qy][qx] = qq * D(qx, qy, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[dy][q] = b(q,dy);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dq = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dq += QQ[qy][qx] * Bt[dx][qx];
-            }
-            QD[qy][dx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dd = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               dd += (QD[qy][dx] * Bt[dy][qy]);
-            }
-            Y(dx, dy, e) += dd;
-         }
-      }
+      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
    });
 }
 
@@ -805,134 +612,18 @@ static void PAMassApply3D(const int NE,
                           const int d1d = 0,
                           const int q1d = 0)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b_.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
    MFEM_FORALL(e, NE,
    {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         double sol_xy[max_Q1D][max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double sol_x[max_Q1D];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] = 0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double s = X(dx,dy,dz,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_x[qx] += B(qx,dx) * s;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy = B(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xy[qy][qx] += wy * sol_x[qx];
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const double wz = B(qz,dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-               }
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         double sol_xy[max_D1D][max_D1D];
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_xy[dy][dx] = 0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double sol_x[max_D1D];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] = 0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double s = sol_xyz[qz][qy][qx];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_x[dx] += Bt(dx,qx) * s;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double wy = Bt(dy,qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_xy[dy][dx] += wy * sol_x[dx];
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            const double wz = Bt(dz,qz);
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
-               }
-            }
-         }
-      }
+      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
    });
 }
 
@@ -953,213 +644,13 @@ static void SmemPAMassApply3D(const int NE,
    constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
    MFEM_VERIFY(D1D <= M1D, "");
    MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   auto b = b_.Read();
+   auto d = d_.Read();
+   auto x = x_.Read();
+   auto y = y_.ReadWrite();
    MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
    {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sDQ[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sDQ;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
-      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-      double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
-      double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
-      double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
-      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
-      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
-      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               X[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-         MFEM_FOREACH_THREAD(dx,x,Q1D)
-         {
-            B[dx][dy] = b(dx,dy);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += X[dz][dy][dx] * B[qx][dx];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
-               {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
-               {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += u[dz];
-            }
-         }
-      }
+      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
    });
 }
 

fem/bilininteg_mass_pa.hpp

@@ -0,0 +1,632 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_MASS_PA_HPP
+#define MFEM_BILININTEG_MASS_PA_HPP
+
+#include "../config/config.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+template <bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void PAMassApply2D_Element(const int e,
+                           const int NE,
+                           const double *b_,
+                           const double *bt_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   const int D1D = d1d;
+   const int Q1D = q1d;
+   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
+   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
+   auto X = ConstDeviceCube(x_, D1D, D1D, NE);
+   auto Y = DeviceCube(y_, D1D, D1D, NE);
+
+   if (!ACCUMULATE)
+   {
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y(dx, dy, e) = 0.0;
+         }
+      }
+   }
+
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
+   double sol_xy[max_Q1D][max_Q1D];
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         sol_xy[qy][qx] = 0.0;
+      }
+   }
+   for (int dy = 0; dy < D1D; ++dy)
+   {
+      double sol_x[max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         sol_x[qy] = 0.0;
+      }
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         const double s = X(dx,dy,e);
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_x[qx] += B(qx,dx)* s;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         const double d2q = B(qy,dy);
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] += d2q * sol_x[qx];
+         }
+      }
+   }
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         sol_xy[qy][qx] *= D(qx,qy,e);
+      }
+   }
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      double sol_x[max_D1D];
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         sol_x[dx] = 0.0;
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         const double s = sol_xy[qy][qx];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] += Bt(dx,qx) * s;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         const double q2d = Bt(dy,qy);
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y(dx,dy,e) += q2d * sol_x[dx];
+         }
+      }
+   }
+}
+
+template<int T_D1D, int T_Q1D, int T_NBZ, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void SmemPAMassApply2D_Element(const int e,
+                               const int NE,
+                               const double *b_,
+                               const double *d_,
+                               const double *x_,
+                               double *y_,
+                               int d1d = 0,
+                               int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
+   auto x = ConstDeviceCube(x_, D1D, D1D, NE);
+   auto Y = DeviceCube(y_, D1D, D1D, NE);
+
+   const int tidz = MFEM_THREAD_ID(z);
+
+   MFEM_SHARED double BBt[MQ1*MD1];
+   double (*B)[MD1] = (double (*)[MD1]) BBt;
+   double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
+   MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
+   MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
+   double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
+   double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
+   double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
+   double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
+
+
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         X[dy][dx] = x(dx,dy,e);
+      }
+   }
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            B[q][dy] = b(q,dy);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double dq = 0.0;
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            dq += X[dy][dx] * B[qx][dx];
+         }
+         DQ[dy][qx] = dq;
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double qq = 0.0;
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            qq += DQ[dy][qx] * B[qy][dy];
+         }
+         QQ[qy][qx] = qq * D(qx, qy, e);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt[dy][q] = b(q,dy);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double dq = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dq += QQ[qy][qx] * Bt[dx][qx];
+         }
+         QD[qy][dx] = dq;
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double dd = 0.0;
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            dd += (QD[qy][dx] * Bt[dy][qy]);
+         }
+         if (ACCUMULATE)
+         {
+            Y(dx, dy, e) += dd;
+         }
+         else
+         {
+            Y(dx, dy, e) = dd;
+         }
+      }
+   }
+}
+
+template <bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void PAMassApply3D_Element(const int e,
+                           const int NE,
+                           const double *b_,
+                           const double *bt_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_,
+                           const int d1d,
+                           const int q1d)
+{
+   const int D1D = d1d;
+   const int Q1D = q1d;
+   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
+   auto D = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
+   auto X = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
+   auto Y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
+
+   if (!ACCUMULATE)
+   {
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx, dy, dz, e) = 0.0;
+            }
+         }
+      }
+   }
+
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
+   double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xyz[qz][qy][qx] = 0.0;
+         }
+      }
+   }
+   for (int dz = 0; dz < D1D; ++dz)
+   {
+      double sol_xy[max_Q1D][max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         double sol_x[max_Q1D];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_x[qx] = 0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double s = X(dx,dy,dz,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_x[qx] += B(qx,dx) * s;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double wy = B(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] += wy * sol_x[qx];
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         const double wz = B(qz,dz);
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+            }
+         }
+      }
+   }
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
+         }
+      }
+   }
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      double sol_xy[max_D1D][max_D1D];
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_xy[dy][dx] = 0;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double sol_x[max_D1D];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] = 0;
+         }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double s = sol_xyz[qz][qy][qx];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] += Bt(dx,qx) * s;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            const double wy = Bt(dy,qy);
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_xy[dy][dx] += wy * sol_x[dx];
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         const double wz = Bt(dz,qz);
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
+            }
+         }
+      }
+   }
+}
+
+template<int T_D1D, int T_Q1D, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void SmemPAMassApply3D_Element(const int e,
+                               const int NE,
+                               const double *b_,
+                               const double *d_,
+                               const double *x_,
+                               double *y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
+{
+   constexpr int D1D = T_D1D ? T_D1D : d1d;
+   constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto d = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
+   auto x = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
+   auto y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
+
+   MFEM_SHARED double sDQ[MQ1*MD1];
+   double (*B)[MD1] = (double (*)[MD1]) sDQ;
+   double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
+   MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
+   MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
+   double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
+   double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
+   double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
+   double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
+   double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
+   double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            X[dz][dy][dx] = x(dx,dy,dz,e);
+         }
+      }
+      MFEM_FOREACH_THREAD(dx,x,Q1D)
+      {
+         B[dx][dy] = b(dx,dy);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] += X[dz][dy][dx] * B[qx][dx];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            DDQ[dz][dy][qx] = u[dz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            DQQ[dz][qy][qx] = u[dz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; qz++)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; qz++)
+         {
+            QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(di,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(q,x,Q1D)
+      {
+         Bt[di][q] = b(q,di);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            QQD[qz][qy][dx] = u[qz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            QDD[qz][dy][dx] = u[qz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            if (ACCUMULATE)
+            {
+               y(dx,dy,dz,e) += u[dz];
+            }
+            else
+            {
+               y(dx,dy,dz,e) = u[dz];
+            }
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif

fem/bilininteg_vecdiffusion.cpp

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"
 
 using namespace std;
@@ -175,43 +176,9 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    MFEM_VERIFY(!VQ && !MQ,
                "Only scalar coefficient supported for partial assembly for VectorDiffusionIntegrator");
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
 
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto Co = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            Co(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
 
    const Array<double> &w = ir->GetWeights();
    const Vector &j = geom->J;

fem/bilininteg_vectorfe.cpp

@@ -11,6 +11,7 @@
 
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
+#include "qspace.hpp"
 #include "gridfunc.hpp"
 
 namespace mfem
@@ -793,140 +794,63 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
    trial_fetype = trial_el->GetDerivType();
    test_fetype = test_el->GetDerivType();
 
-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
-
-   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
-   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
-
-   symmetric = (SMQ || MQ == NULL);
-
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
    const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
    const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
+   if (Q) { coeff.Project(*Q); }
+   else if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
+
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dim*dim);
+
    if ((trial_curl && test_div) || (trial_div && test_curl))
-      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
+      pa_data.SetSize((coeff_dim == 1 ? 1 : dim*dim) * nq * ne,
                       Device::GetMemoryType());
    else
-      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
+      pa_data.SetSize((symmetric ? symmDims : dims*dims) * nq * ne,
                       Device::GetMemoryType());
 
-   Vector coeff;
-
-   auto *qf_c = dynamic_cast<QuadratureFunctionCoefficient*>(Q);
-   if (qf_c)
-   {
-      const QuadratureFunction &qf = qf_c->GetQuadFunction();
-      qf.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction&>(qf), 0);
-   }
-   else
-   {
-      coeff.SetSize(coeffDim * ne * nq);
-      coeff = 1.0;
-      auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-      if (Q || DQ || MQ)
-      {
-         Vector DM(DQ ? coeffDim : 0);
-         DenseMatrix M;
-         DenseSymmetricMatrix SM;
-
-         if (DQ)
-         {
-            MFEM_VERIFY(coeffDim == dim, "");
-         }
-         if (SMQ)
-         {
-            MFEM_VERIFY(SMQ->GetSize() == dim, "");
-            SM.SetSize(dim);
-         }
-         else if (MQ)
-         {
-            MFEM_VERIFY(coeffDim == MQdim, "");
-            MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-            M.SetSize(dim);
-         }
-
-         for (int e=0; e<ne; ++e)
-         {
-            ElementTransformation *tr = mesh->GetElementTransformation(e);
-            for (int p=0; p<nq; ++p)
-            {
-               if (SMQ)
-               {
-                  SMQ->Eval(SM, *tr, ir->IntPoint(p));
-                  int cnt = 0;
-                  for (int i=0; i<dim; ++i)
-                     for (int j=i; j<dim; ++j, ++cnt)
-                     {
-                        coeffh(cnt, p, e) = SM(i,j);
-                     }
-               }
-               else if (MQ)
-               {
-                  MQ->Eval(M, *tr, ir->IntPoint(p));
-
-                  for (int i=0; i<dim; ++i)
-                     for (int j=0; j<dim; ++j)
-                     {
-                        coeffh(j+(i*dim), p, e) = M(i,j);
-                     }
-               }
-               else if (DQ)
-               {
-                  DQ->Eval(DM, *tr, ir->IntPoint(p));
-                  for (int i=0; i<coeffDim; ++i)
-                  {
-                     coeffh(i, p, e) = DM[i];
-                  }
-               }
-               else
-               {
-                  coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-               }
-            }
-         }
-      }
-   }
-
    if (trial_curl && test_curl && dim == 3)
    {
-      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                          coeff, pa_data);
    }
    else if (trial_curl && test_curl && dim == 2)
    {
-      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup2D<2>(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                             coeff, pa_data);
    }
    else if (trial_div && test_div && dim == 3)
    {
-      PAHdivSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
    else if (trial_div && test_div && dim == 2)
    {
-      PAHdivSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup2D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
    else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
             test_fel->GetOrder() == trial_fel->GetOrder())
    {
-      if (coeffDim == 1)
+      if (coeff_dim == 1)
       {
-         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+         PAHcurlL2Setup(nq, coeff_dim, ne, ir->GetWeights(), coeff, pa_data);
       }
       else
       {
          const bool tr = (trial_div && test_curl);
          if (dim == 3)
-            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup3D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
                                geom->J, coeff, pa_data);
          else
-            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup2D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
                                geom->J, coeff, pa_data);
       }
    }
@@ -1168,19 +1092,8 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
 
    pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
 
-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
 
    // Use the same setup functions as VectorFEMassIntegrator.
    if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)

fem/ceed/interface/basis.cpp

@@ -112,7 +112,7 @@ static void InitBasisImpl(const FiniteElementSpace &fes,
    const bool tensor = dynamic_cast<const mfem::TensorBasisElement *>
                        (&fe) != nullptr;
 
-   // Init or retreive key values
+   // Init or retrieve key values
    if (basis_itr == mfem::internal::ceed_basis_map.end())
    {
       if ( tensor )

fem/ceed/interface/coefficient.hpp

@@ -20,6 +20,7 @@
 #include "../../../linalg/dtensor.hpp"
 #include "../../../mesh/mesh.hpp"
 #include "../../gridfunc.hpp"
+#include "../../qfunction.hpp"
 #include "util.hpp"
 #include "ceed.hpp"
 
@@ -121,7 +122,7 @@ void InitCoefficient(mfem::Coefficient *Q, mfem::Mesh &mesh,
       MFEM_VERIFY(qFun.Size() == nq * ne,
                   "Incompatible QuadratureFunction dimension \n");
 
-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
                   "IntegrationRule used within integrator and in"
                   " QuadratureFunction appear to be different");
       qFun.Read();
@@ -195,7 +196,7 @@ void InitCoefficient(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
       MFEM_VERIFY(qFun.Size() == dim * nq * ne,
                   "Incompatible QuadratureFunction dimension \n");
 
-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
                   "IntegrationRule used within integrator and in"
                   " QuadratureFunction appear to be different");
       qFun.Read();
@@ -279,7 +280,7 @@ void InitCoefficientWithIndices(mfem::Coefficient *Q, mfem::Mesh &mesh,
       MFEM_VERIFY(qFun.Size() == nq * ne,
                   "Incompatible QuadratureFunction dimension \n");
 
-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
                   "IntegrationRule used within integrator and in"
                   " QuadratureFunction appear to be different");
       ceedCoeff->coeff.SetSize(nq * nelem);
@@ -369,7 +370,7 @@ void InitCoefficientWithIndices(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
       MFEM_VERIFY(qFun.Size() == dim * nq * ne,
                   "Incompatible QuadratureFunction dimension \n");
 
-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
                   "IntegrationRule used within integrator and in"
                   " QuadratureFunction appear to be different");
       ceedCoeff->coeff.SetSize(dim * nq * nelem);

fem/ceed/interface/restriction.cpp

@@ -232,7 +232,7 @@ void InitRestriction(const FiniteElementSpace &fes,
    RestrKey restr_key(&fes, nelem, P, ncomp, restr_type::Standard);
    auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
 
-   // Init or retreive key values
+   // Init or retrieve key values
    if (restr_itr == mfem::internal::ceed_restr_map.end())
    {
       InitRestrictionImpl(fes, ceed, restr);
@@ -257,7 +257,7 @@ void InitRestrictionWithIndices(const FiniteElementSpace &fes,
    RestrKey restr_key(&fes, nelem, P, ncomp, restr_type::Standard);
    auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
 
-   // Init or retreive key values
+   // Init or retrieve key values
    if (restr_itr == mfem::internal::ceed_restr_map.end())
    {
       InitRestrictionWithIndicesImpl(fes, nelem, indices, ceed, restr);
@@ -281,7 +281,7 @@ void InitCoeffRestrictionWithIndices(const FiniteElementSpace &fes,
    RestrKey restr_key(&fes, nelem, nquads, ncomp, restr_type::Coeff);
    auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
 
-   // Init or retreive key values
+   // Init or retrieve key values
    if (restr_itr == mfem::internal::ceed_restr_map.end())
    {
       InitCoeffRestrictionWithIndicesImpl(fes, nelem, indices, nquads, ncomp,

fem/ceed/solvers/algebraic.cpp

@@ -745,7 +745,7 @@ ParAlgebraicCoarseSpace::ParAlgebraicCoarseSpace(
    ldof_group.SetSize(lsize);
    ldof_group = 0;
 
-   GroupTopology &group_topo = gc_fine->GetGroupTopology();
+   const GroupTopology &group_topo = gc_fine->GetGroupTopology();
    gc = new GroupCommunicator(group_topo);
    Table &group_ldof = gc->GroupLDofTable();
    group_ldof.MakeI(group_ldof_fine.Size());
@@ -822,11 +822,11 @@ HypreParMatrix *ParAlgebraicCoarseSpace::GetProlongationHypreParMatrix()
 
    ParMesh *pmesh = dynamic_cast<ParMesh*>(mesh);
    MFEM_VERIFY(pmesh != NULL, "");
-   Array<HYPRE_Int> dof_offsets, tdof_offsets, tdof_nb_offsets;
-   Array<HYPRE_Int> *offsets[2] = {&dof_offsets, &tdof_offsets};
+   Array<HYPRE_BigInt> dof_offsets, tdof_offsets, tdof_nb_offsets;
+   Array<HYPRE_BigInt> *offsets[2] = {&dof_offsets, &tdof_offsets};
    int lsize = P->Height();
    int ltsize = P->Width();
-   HYPRE_Int loc_sizes[2] = {lsize, ltsize};
+   HYPRE_BigInt loc_sizes[2] = {lsize, ltsize};
    pmesh->GenerateOffsets(2, loc_sizes, offsets);
 
    MPI_Comm comm = pmesh->GetComm();
@@ -870,12 +870,12 @@ HypreParMatrix *ParAlgebraicCoarseSpace::GetProlongationHypreParMatrix()
    HYPRE_Int *j_offd = Memory<HYPRE_Int>(lsize-ltsize);
    int offd_counter;
 
-   HYPRE_Int *cmap   = Memory<HYPRE_Int>(lsize-ltsize);
+   HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(lsize-ltsize);
 
-   HYPRE_Int *col_starts = tdof_offsets;
-   HYPRE_Int *row_starts = dof_offsets;
+   HYPRE_BigInt *col_starts = tdof_offsets;
+   HYPRE_BigInt *row_starts = dof_offsets;
 
-   Array<Pair<HYPRE_Int, int> > cmap_j_offd(lsize-ltsize);
+   Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(lsize-ltsize);
 
    i_diag[0] = i_offd[0] = 0;
    diag_counter = offd_counter = 0;
@@ -909,7 +909,7 @@ HypreParMatrix *ParAlgebraicCoarseSpace::GetProlongationHypreParMatrix()
       i_offd[i_ldof+1] = offd_counter;
    }
 
-   SortPairs<HYPRE_Int, int>(cmap_j_offd, offd_counter);
+   SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);
 
    for (int i = 0; i < offd_counter; i++)
    {

fem/coefficient.cpp

@@ -48,6 +48,31 @@ ElementTransformation *RefinedToCoarse(
    return coarse_T;
 }
 
+void Coefficient::Project(QuadratureFunction &qf)
+{
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   Vector values;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         const int iq_p = qspace.GetPermutedIndex(iel, iq);
+         values[iq_p] = Eval(T, ip);
+      }
+   }
+}
+
+void ConstantCoefficient::Project(QuadratureFunction &qf)
+{
+   qf = constant;
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -135,6 +160,11 @@ double GridFunctionCoefficient::Eval (ElementTransformation &T,
    }
 }
 
+void GridFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf.ProjectGridFunction(*GridF);
+}
+
 void TransformedCoefficient::SetTime(double t)
 {
    if (Q1) { Q1->SetTime(t); }
@@ -203,6 +233,29 @@ void VectorCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
    }
 }
 
+void VectorCoefficient::Project(QuadratureFunction &qf)
+{
+   MFEM_VERIFY(vdim == qf.GetVDim(), "Wrong sizes.");
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   DenseMatrix values;
+   Vector col;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         const int iq_p = qspace.GetPermutedIndex(iel, iq);
+         values.GetColumnReference(iq_p, col);
+         Eval(col, T, ip);
+      }
+   }
+}
+
 void PWVectorCoefficient::InitMap(const Array<int> & attr,
                                   const Array<VectorCoefficient*> & coefs)
 {
@@ -368,6 +421,11 @@ void VectorGridFunctionCoefficient::Eval(
    }
 }
 
+void VectorGridFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf.ProjectGridFunction(*GridFunc);
+}
+
 GradientGridFunctionCoefficient::GradientGridFunctionCoefficient (
    const GridFunction *gf)
    : VectorCoefficient((gf) ?
@@ -517,6 +575,29 @@ void VectorRestrictedCoefficient::Eval(
    }
 }
 
+void MatrixCoefficient::Project(QuadratureFunction &qf, bool transpose)
+{
+   MFEM_VERIFY(qf.GetVDim() == height*width, "Wrong sizes.");
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   DenseMatrix values, matrix;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         const int iq_p = qspace.GetPermutedIndex(iel, iq);
+         matrix.UseExternalData(&values(0, iq_p), height, width);
+         Eval(matrix, T, ip);
+         if (transpose) { matrix.Transpose(); }
+      }
+   }
+}
+
 void PWMatrixCoefficient::InitMap(const Array<int> & attr,
                                   const Array<MatrixCoefficient*> & coefs)
 {
@@ -669,6 +750,31 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
    }
 }
 
+void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
+{
+   const int vdim = qf.GetVDim();
+   MFEM_VERIFY(vdim == height*(height+1)/2, "Wrong sizes.");
+
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   DenseMatrix values;
+   DenseSymmetricMatrix matrix;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         matrix.UseExternalData(&values(0, iq), vdim);
+         Eval(matrix, T, ip);
+      }
+   }
+}
+
+
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
@@ -1437,12 +1543,12 @@ void VectorQuadratureFunctionCoefficient::Eval(Vector &V,
 
    if (index == 0 && vdim == QuadF.GetVDim())
    {
-      QuadF.GetElementValues(T.ElementNo, ip.index, V);
+      QuadF.GetValues(T.ElementNo, ip.index, V);
    }
    else
    {
       Vector temp;
-      QuadF.GetElementValues(T.ElementNo, ip.index, temp);
+      QuadF.GetValues(T.ElementNo, ip.index, temp);
       V.SetSize(vdim);
       for (int i = 0; i < vdim; i++)
       {
@@ -1453,6 +1559,11 @@ void VectorQuadratureFunctionCoefficient::Eval(Vector &V,
    return;
 }
 
+void VectorQuadratureFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf = QuadF;
+}
+
 QuadratureFunctionCoefficient::QuadratureFunctionCoefficient(
    QuadratureFunction &qf) : QuadF(qf)
 {
@@ -1464,8 +1575,210 @@ double QuadratureFunctionCoefficient::Eval(ElementTransformation &T,
 {
    QuadF.HostRead();
    Vector temp(1);
-   QuadF.GetElementValues(T.ElementNo, ip.index, temp);
+   QuadF.GetValues(T.ElementNo, ip.index, temp);
    return temp[0];
 }
 
+void QuadratureFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf = QuadF;
+}
+
+
+CoefficientVector::CoefficientVector(
+   QuadratureSpaceBase &qs_, CoefficientStorage storage_)
+   : Vector(), storage(storage_), vdim(0), qs(qs_), qf(NULL)
+{
+   UseDevice(true);
+}
+
+CoefficientVector::CoefficientVector(Coefficient *coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   if (coeff == NULL)
+   {
+      SetConstant(1.0);
+   }
+   else
+   {
+      Project(*coeff);
+   }
+}
+
+CoefficientVector::CoefficientVector(Coefficient &coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   Project(coeff);
+}
+
+CoefficientVector::CoefficientVector(VectorCoefficient &coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   Project(coeff);
+}
+
+CoefficientVector::CoefficientVector(MatrixCoefficient &coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   Project(coeff);
+}
+
+void CoefficientVector::Project(Coefficient &coeff)
+{
+   vdim = 1;
+   if (auto *const_coeff = dynamic_cast<ConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_coeff->constant);
+   }
+   else if (auto *qf_coeff = dynamic_cast<QuadratureFunctionCoefficient*>(&coeff))
+   {
+      MakeRef(qf_coeff->GetQuadFunction());
+   }
+   else
+   {
+      if (qf == nullptr) { qf = new QuadratureFunction(qs); }
+      qf->SetVDim(1);
+      coeff.Project(*qf);
+      Vector::MakeRef(*qf, 0, qf->Size());
+   }
+}
+
+void CoefficientVector::Project(VectorCoefficient &coeff)
+{
+   vdim = coeff.GetVDim();
+   if (auto *const_coeff = dynamic_cast<VectorConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_coeff->GetVec());
+   }
+   else if (auto *qf_coeff =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&coeff))
+   {
+      MakeRef(qf_coeff->GetQuadFunction());
+   }
+   else
+   {
+      if (qf == nullptr) { qf = new QuadratureFunction(qs, vdim); }
+      qf->SetVDim(vdim);
+      coeff.Project(*qf);
+      Vector::MakeRef(*qf, 0, qf->Size());
+   }
+}
+
+void CoefficientVector::Project(MatrixCoefficient &coeff, bool transpose)
+{
+   if (auto *const_coeff = dynamic_cast<MatrixConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_coeff->GetMatrix());
+   }
+   else if (auto *const_sym_coeff =
+               dynamic_cast<SymmetricMatrixConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_sym_coeff->GetMatrix());
+   }
+   else
+   {
+      auto *sym_coeff = dynamic_cast<SymmetricMatrixCoefficient*>(&coeff);
+      const bool sym = sym_coeff && (storage & CoefficientStorage::SYMMETRIC);
+      const int height = coeff.GetHeight();
+      const int width = coeff.GetWidth();
+      vdim = sym ? height*(height + 1)/2 : width*height;
+
+      if (qf == nullptr) { qf = new QuadratureFunction(qs, vdim); }
+      qf->SetVDim(vdim);
+      if (sym) { sym_coeff->ProjectSymmetric(*qf); }
+      else { coeff.Project(*qf, transpose); }
+      Vector::MakeRef(*qf, 0, qf->Size());
+   }
+}
+
+void CoefficientVector::ProjectTranspose(MatrixCoefficient &coeff)
+{
+   Project(coeff, true);
+}
+
+void CoefficientVector::MakeRef(const QuadratureFunction &qf_)
+{
+   vdim = qf_.GetVDim();
+   const QuadratureSpaceBase *qs2 = qf_.GetSpace();
+   MFEM_CONTRACT_VAR(qs2); // qs2 used only for asserts
+   MFEM_VERIFY(qs2 != NULL, "Invalid QuadratureSpace.")
+   MFEM_VERIFY(qs2->GetMesh() == qs.GetMesh(), "Meshes differ.");
+   MFEM_VERIFY(qs2->GetOrder() == qs.GetOrder(), "Orders differ.");
+   Vector::MakeRef(const_cast<QuadratureFunction&>(qf_), 0, qf_.Size());
+}
+
+void CoefficientVector::SetConstant(double constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   vdim = 1;
+   SetSize(nq);
+   Vector::operator=(constant);
+}
+
+void CoefficientVector::SetConstant(const Vector &constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   vdim = constant.Size();
+   SetSize(nq*vdim);
+   for (int iq = 0; iq < nq; ++iq)
+   {
+      for (int vd = 0; vd<vdim; ++vd)
+      {
+         (*this)[vd + iq*vdim] = constant[vd];
+      }
+   }
+}
+
+void CoefficientVector::SetConstant(const DenseMatrix &constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   const int width = constant.Width();
+   const int height = constant.Height();
+   vdim = width*height;
+   SetSize(nq*vdim);
+   for (int iq = 0; iq < nq; ++iq)
+   {
+      for (int j = 0; j < width; ++j)
+      {
+         for (int i = 0; i < height; ++i)
+         {
+            (*this)[i + j*height + iq*vdim] = constant(i, j);
+         }
+      }
+   }
+}
+
+void CoefficientVector::SetConstant(const DenseSymmetricMatrix &constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   const int height = constant.Height();
+   const bool sym = storage & CoefficientStorage::SYMMETRIC;
+   vdim = sym ? height*(height + 1)/2 : height*height;
+   SetSize(nq*vdim);
+   for (int iq = 0; iq < nq; ++iq)
+   {
+      for (int vd = 0; vd < vdim; ++vd)
+      {
+         const double value = sym ? constant.GetData()[vd] : constant(vd % height,
+                                                                      vd / height);
+         (*this)[vd + iq*vdim] = value;
+      }
+   }
+}
+
+int CoefficientVector::GetVDim() const { return vdim; }
+
+CoefficientVector::~CoefficientVector()
+{
+   delete qf;
+}
+
 }

fem/coefficient.hpp

@@ -23,6 +23,8 @@ namespace mfem
 {
 
 class Mesh;
+class QuadratureSpaceBase;
+class QuadratureFunction;
 
 #ifdef MFEM_USE_MPI
 class ParMesh;
@@ -70,6 +72,10 @@ public:
       return Eval(T, ip);
    }
 
+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   virtual void Project(QuadratureFunction &qf);
+
    virtual ~Coefficient() { }
 };
 
@@ -87,6 +93,9 @@ public:
    virtual double Eval(ElementTransformation &T,
                        const IntegrationPoint &ip)
    { return (constant); }
+
+   /// Fill the QuadratureFunction @a qf with the constant value.
+   void Project(QuadratureFunction &qf);
 };
 
 /** @brief A piecewise constant coefficient with the constants keyed
@@ -274,6 +283,13 @@ public:
    /// Evaluate the coefficient at @a ip.
    virtual double Eval(ElementTransformation &T,
                        const IntegrationPoint &ip);
+
+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// This function uses the efficient QuadratureFunction::ProjectGridFunction
+   /// to fill the QuadratureFunction.
+   virtual void Project(QuadratureFunction &qf);
 };
 
 
@@ -471,6 +487,13 @@ public:
    virtual void Eval(DenseMatrix &M, ElementTransformation &T,
                      const IntegrationRule &ir);
 
+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// The @a vdim of the VectorCoefficient should be equal to the @a vdim of
+   /// the QuadratureFunction.
+   virtual void Project(QuadratureFunction &qf);
+
    virtual ~VectorCoefficient() { }
 };
 
@@ -491,7 +514,7 @@ public:
                      const IntegrationPoint &ip) { V = vec; }
 
    /// Return a reference to the constant vector in this class.
-   const Vector& GetVec() { return vec; }
+   const Vector& GetVec() const { return vec; }
 };
 
 /** @brief A piecewise vector-valued coefficient with the pieces keyed off the
@@ -688,6 +711,13 @@ public:
    virtual void Eval(DenseMatrix &M, ElementTransformation &T,
                      const IntegrationRule &ir);
 
+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// This function uses the efficient QuadratureFunction::ProjectGridFunction
+   /// to fill the QuadratureFunction.
+   virtual void Project(QuadratureFunction &qf);
+
    virtual ~VectorGridFunctionCoefficient() { }
 };
 
@@ -915,6 +945,14 @@ public:
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip) = 0;
 
+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points. The matrix will be transposed or not according to
+   /// the boolean argument @a transpose.
+   ///
+   /// The @a vdim of the QuadratureFunction should be equal to the height times
+   /// the width of the matrix.
+   virtual void Project(QuadratureFunction &qf, bool transpose=false);
+
    /// (DEPRECATED) Evaluate a symmetric matrix coefficient.
    /** @brief Evaluate the upper triangular entries of the matrix coefficient
        in the symmetric case, similarly to Eval. Matrix entry (i,j) is stored
@@ -943,6 +981,8 @@ public:
    /// Evaluate the matrix coefficient at @a ip.
    virtual void Eval(DenseMatrix &M, ElementTransformation &T,
                      const IntegrationPoint &ip) { M = mat; }
+   /// Return a reference to the constant matrix.
+   const DenseMatrix& GetMatrix() { return mat; }
 };
 
 
@@ -1146,6 +1186,8 @@ public:
        can be overridden with the @a own parameter. */
    void Set(int i, int j, Coefficient * c, bool own=true);
 
+   using MatrixCoefficient::Eval;
+
    /// Evaluate coefficient located at (i,j) in the matrix using integration
    /// point @a ip.
    double Eval(int i, int j, ElementTransformation &T, const IntegrationPoint &ip)
@@ -1260,6 +1302,15 @@ public:
    /// Get the size of the matrix.
    int GetSize() const { return height; }
 
+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// @note As opposed to MatrixCoefficient::Project, this function stores only
+   /// the @a symmetric part of the matrix at each quadrature point.
+   ///
+   /// The @a vdim of the coefficient should be equal to height*(height+1)/2.
+   virtual void ProjectSymmetric(QuadratureFunction &qf);
+
    /** @brief Evaluate the matrix coefficient in the element described by @a T
        at the point @a ip, storing the result as a symmetric matrix @a K. */
    /** @note When this method is called, the caller must make sure that the
@@ -1280,6 +1331,9 @@ public:
    virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                      const IntegrationPoint &ip);
 
+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
    virtual ~SymmetricMatrixCoefficient() { }
 };
 
@@ -2049,8 +2103,6 @@ public:
 };
 ///@}
 
-class QuadratureFunction;
-
 /** @brief Vector quadrature function coefficient which requires that the
     quadrature rules used for this vector coefficient be the same as those that
     live within the supplied QuadratureFunction. */
@@ -2075,6 +2127,8 @@ public:
    virtual void Eval(Vector &V, ElementTransformation &T,
                      const IntegrationPoint &ip);
 
+   virtual void Project(QuadratureFunction &qf);
+
    virtual ~VectorQuadratureFunctionCoefficient() { }
 };
 
@@ -2094,9 +2148,123 @@ public:
 
    virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
 
+   virtual void Project(QuadratureFunction &qf);
+
    virtual ~QuadratureFunctionCoefficient() { }
 };
 
+/// Flags that determine what storage optimizations to use in CoefficientVector
+enum class CoefficientStorage : int
+{
+   FULL = 0, ///< Store the coefficient as a full QuadratureFunction.
+   CONSTANTS = 1 << 0, ///< Store constants using only @a vdim entries.
+   SYMMETRIC = 1 << 1, ///< Store the triangular part of symmetric matrices.
+   COMPRESSED = CONSTANTS | SYMMETRIC ///< Enable all above compressions.
+};
+
+inline CoefficientStorage operator|(CoefficientStorage a, CoefficientStorage b)
+{
+   return CoefficientStorage(int(a) | int(b));
+}
+
+inline int operator&(CoefficientStorage a, CoefficientStorage b)
+{
+   return int(a) & int(b);
+}
+
+
+/// @brief Class to represent a coefficient evaluated at quadrature points.
+///
+/// In the general case, a CoefficientVector is the same as a QuadratureFunction
+/// with a coefficient projected onto it.
+///
+/// This class allows for some "compression" of the coefficient data, according
+/// to the storage flags given by CoefficientStorage. For example, constant
+/// coefficients can be stored using only @a vdim values, and symmetric matrices
+/// can be stored using e.g. the upper triangular part of the matrix.
+class CoefficientVector : public Vector
+{
+protected:
+   CoefficientStorage storage; ///< Storage optimizations (see CoefficientStorage).
+   int vdim; ///< Number of values per quadrature point.
+   QuadratureSpaceBase &qs; ///< Associated QuadratureSpaceBase.
+   QuadratureFunction *qf; ///< Internal QuadratureFunction (owned, may be NULL).
+public:
+   /// Create an empty CoefficientVector.
+   CoefficientVector(QuadratureSpaceBase &qs_,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given Coefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// If @a coeff is NULL, it will be interpreted as a constant with value one.
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(Coefficient *coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given Coefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(Coefficient &coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given VectorCoefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(VectorCoefficient &coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given MatrixCoefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(MatrixCoefficient &coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Evaluate the given Coefficient at the quadrature points defined by
+   /// @ref qs.
+   void Project(Coefficient &coeff);
+
+   /// @brief Evaluate the given VectorCoefficient at the quadrature points
+   /// defined by @ref qs.
+   ///
+   /// @sa CoefficientVector for a description of the @a compress argument.
+   void Project(VectorCoefficient &coeff);
+
+   /// @brief Evaluate the given MatrixCoefficient at the quadrature points
+   /// defined by @ref qs.
+   ///
+   /// @sa CoefficientVector for a description of the @a compress argument.
+   void Project(MatrixCoefficient &coeff, bool transpose=false);
+
+   /// @brief Project the tranpose of @a coeff.
+   ///
+   /// @sa Project(MatrixCoefficient&, QuadratureSpace&, bool, bool)
+   void ProjectTranspose(MatrixCoefficient &coeff);
+
+   /// Make this vector a reference to the given QuadratureFunction.
+   void MakeRef(const QuadratureFunction &qf_);
+
+   /// Set this vector to the given constant.
+   void SetConstant(double constant);
+
+   /// Set this vector to the given constant vector.
+   void SetConstant(const Vector &constant);
+
+   /// Set this vector to the given constant matrix.
+   void SetConstant(const DenseMatrix &constant);
+
+   /// Set this vector to the given constant symmetric matrix.
+   void SetConstant(const DenseSymmetricMatrix &constant);
+
+   /// Return the number of values per quadrature point.
+   int GetVDim() const;
+
+   ~CoefficientVector();
+};
+
 /** @brief Compute the Lp norm of a function f.
     \f$ \| f \|_{Lp} = ( \int_\Omega | f |^p d\Omega)^{1/p} \f$ */
 double ComputeLpNorm(double p, Coefficient &coeff, Mesh &mesh,

fem/datacollection.cpp

@@ -442,7 +442,7 @@ void VisItDataCollection::RegisterQField(const std::string& name,
    {
       int locLOD = GlobGeometryRefiner.GetRefinementLevelFromElems(
                       mesh->GetElementBaseGeometry(e),
-                      qf->GetElementIntRule(e).GetNPoints());
+                      qf->GetIntRule(e).GetNPoints());
 
       LOD = std::max(LOD,locLOD);
    }

fem/datacollection.hpp

@@ -14,6 +14,7 @@
 
 #include "../config/config.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #ifdef MFEM_USE_MPI
 #include "pgridfunc.hpp"
 #endif

fem/dgmassinv.cpp

@@ -0,0 +1,315 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "dgmassinv.hpp"
+#include "bilinearform.hpp"
+#include "dgmassinv_kernels.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_orig, Coefficient *coeff,
+                             const IntegrationRule *ir,
+                             int btype)
+   : Solver(fes_orig.GetTrueVSize()),
+     fec(fes_orig.GetMaxElementOrder(),
+         fes_orig.GetMesh()->Dimension(),
+         btype,
+         fes_orig.GetFE(0)->GetMapType()),
+     fes(fes_orig.GetMesh(), &fec)
+{
+   MFEM_VERIFY(fes.IsDGSpace(), "Space must be DG.");
+   MFEM_VERIFY(!fes.IsVariableOrder(), "Variable orders not supported.");
+
+   const int btype_orig =
+      static_cast<const L2_FECollection*>(fes_orig.FEColl())->GetBasisType();
+
+   if (btype_orig == btype)
+   {
+      // No change of basis required
+      d2q = nullptr;
+   }
+   else
+   {
+      // original basis to solver basis
+      const auto mode = DofToQuad::TENSOR;
+      d2q = &fes_orig.GetFE(0)->GetDofToQuad(fes.GetFE(0)->GetNodes(), mode);
+
+      int n = d2q->ndof;
+      Array<double> B_inv = d2q->B; // deep copy
+      Array<int> ipiv(n);
+      // solver basis to original
+      LUFactors lu(B_inv.HostReadWrite(), ipiv.HostWrite());
+      lu.Factor(n);
+      B_.SetSize(n*n);
+      lu.GetInverseMatrix(n, B_.HostWrite());
+      Bt_.SetSize(n*n);
+      DenseMatrix B_matrix(B_.HostReadWrite(), n, n);
+      DenseMatrix Bt_matrix(Bt_.HostWrite(), n, n);
+      Bt_matrix.Transpose(B_matrix);
+   }
+
+   if (coeff) { m = new MassIntegrator(*coeff, ir); }
+   else { m = new MassIntegrator(ir); }
+
+   diag_inv.SetSize(height);
+   // Workspace vectors used for CG
+   r_.SetSize(height);
+   d_.SetSize(height);
+   z_.SetSize(height);
+   // Only need transformed RHS if basis is different
+   if (btype_orig != btype) { b2_.SetSize(height); }
+
+   M = new BilinearForm(&fes);
+   M->AddDomainIntegrator(m); // M assumes ownership of m
+   M->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+
+   // Assemble the bilinear form and its diagonal (for preconditioning).
+   Update();
+}
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                             int btype)
+   : DGMassInverse(fes_, &coeff, nullptr, btype) { }
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                             const IntegrationRule &ir, int btype)
+   : DGMassInverse(fes_, &coeff, &ir, btype) { }
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_,
+                             const IntegrationRule &ir, int btype)
+   : DGMassInverse(fes_, nullptr, &ir, btype) { }
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_, int btype)
+   : DGMassInverse(fes_, nullptr, nullptr, btype) { }
+
+void DGMassInverse::SetOperator(const Operator &op)
+{
+   MFEM_ABORT("SetOperator not supported with DGMassInverse.")
+}
+
+void DGMassInverse::SetRelTol(const double rel_tol_) { rel_tol = rel_tol_; }
+
+void DGMassInverse::SetAbsTol(const double abs_tol_) { abs_tol = abs_tol_; }
+
+void DGMassInverse::SetMaxIter(const double max_iter_) { max_iter = max_iter_; }
+
+void DGMassInverse::Update()
+{
+   M->Assemble();
+   M->AssembleDiagonal(diag_inv);
+   internal::MakeReciprocal(diag_inv.Size(), diag_inv.ReadWrite());
+}
+
+DGMassInverse::~DGMassInverse()
+{
+   delete M;
+}
+
+template<int DIM, int D1D, int Q1D>
+void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
+{
+   using namespace internal; // host/device kernel functions
+
+   const int NE = fes.GetNE();
+   const int d1d = m->dofs1D;
+   const int q1d = m->quad1D;
+
+   const int ND = static_cast<int>(pow(d1d, DIM));
+
+   const auto B = m->maps->B.Read();
+   const auto Bt = m->maps->Bt.Read();
+   const auto pa_data = m->pa_data.Read();
+   const auto dinv = diag_inv.Read();
+   auto r = r_.Write();
+   auto d = d_.Write();
+   auto z = z_.Write();
+   auto u = u_.ReadWrite();
+
+   const double RELTOL = rel_tol;
+   const double ABSTOL = abs_tol;
+   const double MAXIT = max_iter;
+   const bool IT_MODE = iterative_mode;
+   const bool CHANGE_BASIS = (d2q != nullptr);
+
+   // b is the right-hand side (if no change of basis, this just points to the
+   // incoming RHS vector, if we have to change basis, this points to the
+   // internal b2 vector where we put the transformed RHS)
+   const double *b;
+   // the following are non-null if we have to change basis
+   double *b2 = nullptr; // non-const access to b2
+   const double *b_orig = nullptr; // RHS vector in "original" basis
+   const double *d2q_B = nullptr; // matrix to transform initial guess
+   const double *q2d_B = nullptr; // matrix to transform solution
+   const double *q2d_Bt = nullptr; // matrix to transform RHS
+   if (CHANGE_BASIS)
+   {
+      d2q_B = d2q->B.Read();
+      q2d_B = B_.Read();
+      q2d_Bt = Bt_.Read();
+
+      b2 = b2_.Write();
+      b_orig = b_.Read();
+      b = b2;
+   }
+   else
+   {
+      b = b_.Read();
+   }
+
+   constexpr int NB = Q1D ? Q1D : 1; // block size
+
+   MFEM_FORALL_2D(e, NE, NB, NB, 1,
+   {
+      constexpr int NB = Q1D ? Q1D : 1; // redefine here for some compilers
+
+      // Perform change of basis if needed
+      if (CHANGE_BASIS)
+      {
+         // Transform RHS
+         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
+         if (IT_MODE)
+         {
+            // Transform initial guess
+            DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, d2q_B, u, u, d1d);
+         }
+      }
+
+      const int tid = MFEM_THREAD_ID(x) + NB*MFEM_THREAD_ID(y);
+
+      // Compute first residual
+      if (IT_MODE)
+      {
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, u, r, d1d, q1d);
+         DGMassAxpy(e, NE, ND, 1.0, b, -1.0, r, r); // r = b - r
+      }
+      else
+      {
+         // if not in iterative mode, use zero initial guess
+         const int BX = MFEM_THREAD_SIZE(x);
+         const int BY = MFEM_THREAD_SIZE(y);
+         const int bxy = BX*BY;
+         const auto B = ConstDeviceMatrix(b, ND, NE);
+         auto U = DeviceMatrix(u, ND, NE);
+         auto R = DeviceMatrix(r, ND, NE);
+         for (int i = tid; i < ND; i += bxy)
+         {
+            U(i, e) = 0.0;
+            R(i, e) = B(i, e);
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      DGMassPreconditioner(e, NE, ND, dinv, r, z);
+      DGMassAxpy(e, NE, ND, 1.0, z, 0.0, z, d); // d = z
+
+      double nom = DGMassDot<NB>(e, NE, ND, d, r);
+      if (nom < 0.0) { return; /* Not positive definite */ }
+      double r0 = fmax(nom*RELTOL*RELTOL, ABSTOL*ABSTOL);
+      if (nom <= r0) { return; /* Converged */ }
+
+      DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d);
+      double den = DGMassDot<NB>(e, NE, ND, z, d);
+      if (den <= 0.0)
+      {
+         DGMassDot<NB>(e, NE, ND, d, d);
+         // d2 > 0 => not positive definite
+         if (den == 0.0) { return; }
+      }
+
+      // start iteration
+      int i = 1;
+      while (true)
+      {
+         const double alpha = nom/den;
+         DGMassAxpy(e, NE, ND, 1.0, u, alpha, d, u); // u = u + alpha*d
+         DGMassAxpy(e, NE, ND, 1.0, r, -alpha, z, r); // r = r - alpha*A*d
+
+         DGMassPreconditioner(e, NE, ND, dinv, r, z);
+
+         double betanom = DGMassDot<NB>(e, NE, ND, r, z);
+         if (betanom < 0.0) { return; /* Not positive definite */ }
+         if (betanom <= r0) { break; /* Converged */ }
+
+         if (++i > MAXIT) { break; }
+
+         const double beta = betanom/nom;
+         DGMassAxpy(e, NE, ND, 1.0, z, beta, d, d); // d = z + beta*d
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d); // z = A d
+         den = DGMassDot<NB>(e, NE, ND, d, z);
+         if (den <= 0.0)
+         {
+            DGMassDot<NB>(e, NE, ND, d, d);
+            // d2 > 0 => not positive definite
+            if (den == 0.0) { break; }
+         }
+         nom = betanom;
+      }
+
+      if (CHANGE_BASIS)
+      {
+         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_B, u, u, d1d);
+      }
+   });
+}
+
+void DGMassInverse::Mult(const Vector &Mu, Vector &u) const
+{
+   // Dispatch to templated version based on dim, d1d, and q1d.
+   const int dim = fes.GetMesh()->Dimension();
+   const int d1d = m->dofs1D;
+   const int q1d = m->quad1D;
+
+   const int id = (d1d << 4) | q1d;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x11: return DGMassCGIteration<2,1,1>(Mu, u);
+         case 0x22: return DGMassCGIteration<2,2,2>(Mu, u);
+         case 0x33: return DGMassCGIteration<2,3,3>(Mu, u);
+         case 0x35: return DGMassCGIteration<2,3,5>(Mu, u);
+         case 0x44: return DGMassCGIteration<2,4,4>(Mu, u);
+         case 0x46: return DGMassCGIteration<2,4,6>(Mu, u);
+         case 0x55: return DGMassCGIteration<2,5,5>(Mu, u);
+         case 0x57: return DGMassCGIteration<2,5,7>(Mu, u);
+         case 0x66: return DGMassCGIteration<2,6,6>(Mu, u);
+         case 0x68: return DGMassCGIteration<2,6,8>(Mu, u);
+         default: return DGMassCGIteration<2>(Mu, u); // Fallback
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x22: return DGMassCGIteration<3,2,2>(Mu, u);
+         case 0x23: return DGMassCGIteration<3,2,3>(Mu, u);
+         case 0x33: return DGMassCGIteration<3,3,3>(Mu, u);
+         case 0x34: return DGMassCGIteration<3,3,4>(Mu, u);
+         case 0x35: return DGMassCGIteration<3,3,5>(Mu, u);
+         case 0x44: return DGMassCGIteration<3,4,4>(Mu, u);
+         case 0x45: return DGMassCGIteration<3,4,5>(Mu, u);
+         case 0x46: return DGMassCGIteration<3,4,6>(Mu, u);
+         case 0x48: return DGMassCGIteration<3,4,8>(Mu, u);
+         case 0x55: return DGMassCGIteration<3,5,5>(Mu, u);
+         case 0x56: return DGMassCGIteration<3,5,6>(Mu, u);
+         case 0x57: return DGMassCGIteration<3,5,7>(Mu, u);
+         case 0x58: return DGMassCGIteration<3,5,8>(Mu, u);
+         case 0x66: return DGMassCGIteration<3,6,6>(Mu, u);
+         case 0x67: return DGMassCGIteration<3,6,7>(Mu, u);
+         default: return DGMassCGIteration<3>(Mu, u); // Fallback
+      }
+   }
+}
+
+} // namespace mfem

fem/dgmassinv.hpp

@@ -0,0 +1,112 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DGMASSINV_HPP
+#define MFEM_DGMASSINV_HPP
+
+#include "../linalg/operator.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+/// @brief Solver for the discontinuous Galerkin mass matrix.
+///
+/// This class performs a @a local (diagonally preconditioned) conjugate
+/// gradient iteration for each element. Optionally, a change of basis is
+/// performed to iterate on a better-conditioned system. This class fully
+/// supports execution on device (GPU).
+class DGMassInverse : public Solver
+{
+protected:
+   DG_FECollection fec; ///< FE collection in requested basis.
+   FiniteElementSpace fes; ///< FE space in requested basis.
+   const DofToQuad *d2q; ///< Change of basis. Not owned.
+   Array<double> B_; ///< Inverse of change of basis.
+   Array<double> Bt_; ///< Inverse of change of basis, transposed.
+   class BilinearForm *M; ///< Mass bilinear form, owned.
+   class MassIntegrator *m; ///< Mass integrator, owned by the form @ref M.
+   Vector diag_inv; ///< Jacobi preconditioner.
+   double rel_tol = 1e-12; ///< Relative CG tolerance.
+   double abs_tol = 1e-12; ///< Absolute CG tolerance.
+   int max_iter = 100; ///< Maximum number of CG iterations;
+
+   /// @name Intermediate vectors needed for CG three-term recurrence.
+   ///@{
+   mutable Vector r_, d_, z_, b2_;
+   ///@}
+
+   /// @brief Protected constructor, used internally.
+   ///
+   /// Custom coefficient and integration rule are used if @a coeff and @a ir
+   /// are non-NULL.
+   DGMassInverse(FiniteElementSpace &fes_, Coefficient *coeff,
+                 const IntegrationRule *ir, int btype);
+public:
+   /// @brief Construct the DG inverse mass operator for @a fes_.
+   ///
+   /// The basis type @a btype determines which basis should be used internally
+   /// in the solver. This <b>does not</b> have to be the same basis as @a fes_.
+   /// The best choice is typically BasisType::GaussLegendre because it is
+   /// well-preconditioned by its diagonal.
+   ///
+   /// The solution and right-hand side used for the solver are not affected by
+   /// this basis (they correspond to the basis of @a fes_). @a btype is only
+   /// used internally, and only has an effect on the convergence rate.
+   DGMassInverse(FiniteElementSpace &fes_, int btype=BasisType::GaussLegendre);
+   /// @brief Construct the DG inverse mass operator for @a fes_ with
+   /// Coefficient @a coeff.
+   ///
+   /// @sa DGMassInverse(FiniteElementSpace&, int) for information about @a
+   /// btype.
+   DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                 int btype=BasisType::GaussLegendre);
+   /// @brief Construct the DG inverse mass operator for @a fes_ with
+   /// Coefficient @a coeff and IntegrationRule @a ir.
+   ///
+   /// @sa DGMassInverse(FiniteElementSpace&, int) for information about @a
+   /// btype.
+   DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                 const IntegrationRule &ir, int btype=BasisType::GaussLegendre);
+   /// @brief Construct the DG inverse mass operator for @a fes_ with
+   /// IntegrationRule @a ir.
+   ///
+   /// @sa DGMassInverse(FiniteElementSpace&, int) for information about @a
+   /// btype.
+   DGMassInverse(FiniteElementSpace &fes_, const IntegrationRule &ir,
+                 int btype=BasisType::GaussLegendre);
+   /// @brief Solve the system M b = u.
+   ///
+   /// If @ref iterative_mode is @a true, @a u is used as an initial guess.
+   void Mult(const Vector &b, Vector &u) const;
+   /// Not implemented. Aborts.
+   void SetOperator(const Operator &op);
+   /// Set the relative tolerance.
+   void SetRelTol(const double rel_tol_);
+   /// Set the absolute tolerance.
+   void SetAbsTol(const double abs_tol_);
+   /// Set the maximum number of iterations.
+   void SetMaxIter(const double max_iter_);
+   /// Recompute operator and preconditioner (when coefficient or mesh changes).
+   void Update();
+
+   ~DGMassInverse();
+
+   /// @brief Solve the system M b = u. <b>Not part of the public interface.</b>
+   /// @note This member function must be public because it contains an
+   /// MFEM_FORALL kernel (nvcc limitation)
+   template<int DIM, int D1D = 0, int Q1D = 0>
+   void DGMassCGIteration(const Vector &b_, Vector &u_) const;
+};
+
+} // namespace mfem
+
+#endif

fem/dgmassinv_kernels.hpp

@@ -0,0 +1,295 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DGMASSINV_KERNELS_HPP
+#define MFEM_DGMASSINV_KERNELS_HPP
+
+#include "bilininteg_mass_pa.hpp"
+#include "../linalg/kernels.hpp"
+#include "kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+void MakeReciprocal(int n, double *x)
+{
+   MFEM_FORALL(i, n, x[i] = 1.0/x[i]; );
+}
+
+template <int DIM, int D1D, int Q1D>
+MFEM_HOST_DEVICE inline
+void DGMassApply(const int e,
+                 const int NE,
+                 const double *B,
+                 const double *Bt,
+                 const double *pa_data,
+                 const double *x,
+                 double *y,
+                 const int d1d = 0,
+                 const int q1d = 0)
+{
+   constexpr bool use_smem = (D1D > 0 && Q1D > 0);
+   constexpr bool ACCUM = false;
+   constexpr int NBZ = 1;
+   if (use_smem)
+   {
+      // cannot specialize functions below with D1D or Q1D equal to zero
+      // (this branch only runs with D1D and Q1D are both positive)
+      constexpr int TD1D = D1D ? D1D : 1;
+      constexpr int TQ1D = Q1D ? Q1D : 1;
+      if (DIM == 2)
+      {
+         SmemPAMassApply2D_Element<TD1D,TQ1D,NBZ,ACCUM>(e, NE, B, pa_data, x, y);
+      }
+      else if (DIM == 3)
+      {
+         SmemPAMassApply3D_Element<TD1D,TQ1D,ACCUM>(e, NE, B, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("Unsupported dimension.");
+      }
+   }
+   else
+   {
+      if (DIM == 2)
+      {
+         PAMassApply2D_Element<ACCUM>(e, NE, B, Bt, pa_data, x, y, d1d, q1d);
+      }
+      else if (DIM == 3)
+      {
+         PAMassApply3D_Element<ACCUM>(e, NE, B, Bt, pa_data, x, y, d1d, q1d);
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("Unsupported dimension.");
+      }
+   }
+}
+
+MFEM_HOST_DEVICE inline
+void DGMassPreconditioner(const int e,
+                          const int NE,
+                          const int ND,
+                          const double *dinv,
+                          const double *x,
+                          double *y)
+{
+   const auto X = ConstDeviceMatrix(x, ND, NE);
+   const auto D = ConstDeviceMatrix(dinv, ND, NE);
+   auto Y = DeviceMatrix(y, ND, NE);
+
+   const int tid = MFEM_THREAD_ID(x) + MFEM_THREAD_SIZE(x)*MFEM_THREAD_ID(y);
+   const int bxy = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
+
+   for (int i = tid; i < ND; i += bxy)
+   {
+      Y(i, e) = D(i, e)*X(i, e);
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void DGMassAxpy(const int e,
+                const int NE,
+                const int ND,
+                const double a,
+                const double *x,
+                const double b,
+                const double *y,
+                double *z)
+{
+   const auto X = ConstDeviceMatrix(x, ND, NE);
+   const auto Y = ConstDeviceMatrix(y, ND, NE);
+   auto Z = DeviceMatrix(z, ND, NE);
+
+   const int tid = MFEM_THREAD_ID(x) + MFEM_THREAD_SIZE(x)*MFEM_THREAD_ID(y);
+   const int bxy = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
+
+   for (int i = tid; i < ND; i += bxy)
+   {
+      Z(i, e) = a*X(i, e) + b*Y(i, e);
+   }
+   MFEM_SYNC_THREAD;
+}
+
+template <int NB>
+MFEM_HOST_DEVICE inline
+double DGMassDot(const int e,
+                 const int NE,
+                 const int ND,
+                 const double *x,
+                 const double *y)
+{
+   const auto X = ConstDeviceMatrix(x, ND, NE);
+   const auto Y = ConstDeviceMatrix(y, ND, NE);
+
+   const int tid = MFEM_THREAD_ID(x) + MFEM_THREAD_SIZE(x)*MFEM_THREAD_ID(y);
+   const int bxy = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
+
+   MFEM_SHARED double s_dot[NB*NB];
+   s_dot[tid] = 0.0;
+
+   for (int i = tid; i < ND; i += bxy) { s_dot[tid] += X(i,e)*Y(i,e); }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 512 && tid + 512 < bxy) { s_dot[tid] += s_dot[tid + 512]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 256 && tid < 256 && tid + 256 < bxy) { s_dot[tid] += s_dot[tid + 256]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 128 && tid < 128 && tid + 128 < bxy) { s_dot[tid] += s_dot[tid + 128]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 64 && tid < 64 && tid + 64 < bxy) { s_dot[tid] += s_dot[tid + 64]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 32 && tid < 32 && tid + 32 < bxy) { s_dot[tid] += s_dot[tid + 32]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 16 && tid < 16 && tid + 16 < bxy) { s_dot[tid] += s_dot[tid + 16]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 8 && tid < 8 && tid + 8 < bxy) { s_dot[tid] += s_dot[tid + 8]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 4 && tid < 4 && tid + 4 < bxy) { s_dot[tid] += s_dot[tid + 4]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 2 && tid < 2 && tid + 2 < bxy) { s_dot[tid] += s_dot[tid + 2]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 1 && tid < 1 && tid + 1 < bxy) { s_dot[tid] += s_dot[tid + 1]; }
+   MFEM_SYNC_THREAD;
+
+   return s_dot[0];
+}
+
+template<int T_D1D = 0, int MAX_D1D = 0>
+MFEM_HOST_DEVICE inline
+void DGMassBasis2D(const int e,
+                   const int NE,
+                   const double *b_,
+                   const double *x_,
+                   double *y_,
+                   const int d1d = 0)
+{
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   const int D1D = T_D1D ? T_D1D : d1d;
+
+   const auto b = Reshape(b_, D1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, NE);
+   auto y = Reshape(y_, D1D, D1D, NE);
+
+   MFEM_SHARED double sB[MD1*MD1];
+   MFEM_SHARED double sm0[MD1*MD1];
+   MFEM_SHARED double sm1[MD1*MD1];
+
+   kernels::internal::LoadB<MD1,MD1>(D1D,D1D,b,sB);
+
+   ConstDeviceMatrix B(sB, D1D,D1D);
+   DeviceMatrix DD(sm0, MD1, MD1);
+   DeviceMatrix DQ(sm1, MD1, MD1);
+   DeviceMatrix QQ(sm0, MD1, MD1);
+
+   kernels::internal::LoadX(e,D1D,x,DD);
+   kernels::internal::EvalX(D1D,D1D,B,DD,DQ);
+   kernels::internal::EvalY(D1D,D1D,B,DQ,QQ);
+   MFEM_SYNC_THREAD; // sync here to allow in-place evaluations
+   MFEM_FOREACH_THREAD(qy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,D1D)
+      {
+         y(qx,qy,e) = QQ(qx,qy);
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+template<int T_D1D = 0, int MAX_D1D = 0>
+MFEM_HOST_DEVICE inline
+void DGMassBasis3D(const int e,
+                   const int NE,
+                   const double *b_,
+                   const double *x_,
+                   double *y_,
+                   const int d1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+
+   const auto b = Reshape(b_, D1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, NE);
+   auto y = Reshape(y_, D1D, D1D, D1D, NE);
+
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+
+   MFEM_SHARED double sB[MD1*MD1];
+   MFEM_SHARED double sm0[MD1*MD1*MD1];
+   MFEM_SHARED double sm1[MD1*MD1*MD1];
+
+   kernels::internal::LoadB<MD1,MD1>(D1D,D1D,b,sB);
+
+   ConstDeviceMatrix B(sB, D1D,D1D);
+   DeviceCube DDD(sm0, MD1,MD1,MD1);
+   DeviceCube DDQ(sm1, MD1,MD1,MD1);
+   DeviceCube DQQ(sm0, MD1,MD1,MD1);
+   DeviceCube QQQ(sm1, MD1,MD1,MD1);
+
+   kernels::internal::LoadX(e,D1D,x,DDD);
+   kernels::internal::EvalX(D1D,D1D,B,DDD,DDQ);
+   kernels::internal::EvalY(D1D,D1D,B,DDQ,DQQ);
+   kernels::internal::EvalZ(D1D,D1D,B,DQQ,QQQ);
+   MFEM_SYNC_THREAD; // sync here to allow in-place evaluation
+   MFEM_FOREACH_THREAD(qz,z,D1D)
+   {
+      MFEM_FOREACH_THREAD(qy,y,D1D)
+      {
+         for (int qx = 0; qx < D1D; ++qx)
+         {
+            y(qx,qy,qz,e) = QQQ(qz,qy,qx);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+template<int DIM, int T_D1D = 0, int MAX_D1D = 0>
+MFEM_HOST_DEVICE inline
+void DGMassBasis(const int e,
+                 const int NE,
+                 const double *b_,
+                 const double *x_,
+                 double *y_,
+                 const int d1d = 0)
+{
+   if (DIM == 2)
+   {
+      DGMassBasis2D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
+   }
+   else if (DIM == 3)
+   {
+      DGMassBasis3D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("Dimension not supported.");
+   }
+}
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif

fem/fe_coll.hpp

@@ -53,8 +53,12 @@ public:
 
    virtual int DofForGeometry(Geometry::Type GeomType) const = 0;
 
-   /** @brief Returns an array, say p, that maps a local permuted index i to
-       a local base index: base_i = p[i]. */
+   /** @brief Returns an array, say p, that maps a local permuted index i to a
+       local base index: base_i = p[i].
+
+       @note Only provides information about interior dofs. See
+       FiniteElementCollection::SubDofOrder if interior \a and boundary dof
+       order is needed. */
    virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
                                              int Or) const = 0;
 
@@ -95,10 +99,10 @@ public:
    | RT_ValTrace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
    | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
    | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinous L2 elements |
-   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinous L2 elements |
-   | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinous L2 elements |
-   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinous L2 elements |
+   | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
+   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
+   | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
+   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
    | DG_Iface_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
    | DG_Iface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
    | DG_IntIface_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |

fem/fem.hpp

@@ -45,6 +45,7 @@
 #include "multigrid.hpp"
 #include "ceed/solvers/algebraic.hpp"
 #include "lor/lor.hpp"
+#include "dgmassinv.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/fespace.cpp

@@ -1258,7 +1258,7 @@ int FiniteElementSpace::GetNConformingDofs() const
    return P ? (P->Width() / vdim) : ndofs;
 }
 
-const Operator *FiniteElementSpace::GetElementRestriction(
+const ElementRestrictionOperator *FiniteElementSpace::GetElementRestriction(
    ElementDofOrdering e_ordering) const
 {
    // Check if we have a discontinuous space using the FE collection:
@@ -1273,7 +1273,7 @@ const Operator *FiniteElementSpace::GetElementRestriction(
          // The output E-vector layout is: ND x VDIM x NE.
          L2E_nat.Reset(new L2ElementRestriction(*this));
       }
-      return L2E_nat.Ptr();
+      return L2E_nat.Is<ElementRestrictionOperator>();
    }
    if (e_ordering == ElementDofOrdering::LEXICOGRAPHIC)
    {
@@ -1281,14 +1281,14 @@ const Operator *FiniteElementSpace::GetElementRestriction(
       {
          L2E_lex.Reset(new ElementRestriction(*this, e_ordering));
       }
-      return L2E_lex.Ptr();
+      return L2E_lex.Is<ElementRestrictionOperator>();
    }
    // e_ordering == ElementDofOrdering::NATIVE
    if (L2E_nat.Ptr() == NULL)
    {
       L2E_nat.Reset(new ElementRestriction(*this, e_ordering));
    }
-   return L2E_nat.Ptr();
+   return L2E_nat.Is<ElementRestrictionOperator>();
 }
 
 const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
@@ -3613,58 +3613,4 @@ FiniteElementCollection *FiniteElementSpace::Load(Mesh *m, std::istream &input)
    return r_fec;
 }
 
-
-void QuadratureSpace::Construct()
-{
-   // protected method
-   int offset = 0;
-   const int num_elem = mesh->GetNE();
-   element_offsets = new int[num_elem + 1];
-   for (int g = 0; g < Geometry::NumGeom; g++)
-   {
-      int_rule[g] = NULL;
-   }
-   for (int i = 0; i < num_elem; i++)
-   {
-      element_offsets[i] = offset;
-      int geom = mesh->GetElementBaseGeometry(i);
-      if (int_rule[geom] == NULL)
-      {
-         int_rule[geom] = &IntRules.Get(geom, order);
-      }
-      offset += int_rule[geom]->GetNPoints();
-   }
-   element_offsets[num_elem] = size = offset;
-}
-
-QuadratureSpace::QuadratureSpace(Mesh *mesh_, std::istream &in)
-   : mesh(mesh_)
-{
-   const char *msg = "invalid input stream";
-   string ident;
-
-   in >> ident; MFEM_VERIFY(ident == "QuadratureSpace", msg);
-   in >> ident; MFEM_VERIFY(ident == "Type:", msg);
-   in >> ident;
-   if (ident == "default_quadrature")
-   {
-      in >> ident; MFEM_VERIFY(ident == "Order:", msg);
-      in >> order;
-   }
-   else
-   {
-      MFEM_ABORT("unknown QuadratureSpace type: " << ident);
-      return;
-   }
-
-   Construct();
-}
-
-void QuadratureSpace::Save(std::ostream &os) const
-{
-   os << "QuadratureSpace\n"
-      << "Type: default_quadrature\n"
-      << "Order: " << order << '\n';
-}
-
 } // namespace mfem

fem/fespace.hpp

@@ -47,6 +47,14 @@ public:
    static void DofsToVDofs(int ndofs, int vdim, Array<int> &dofs);
 };
 
+/// @brief Type describing possible layouts for Q-vectors.
+/// @sa QuadratureInterpolator and FaceQuadratureInterpolator.
+enum class QVectorLayout
+{
+   byNODES,  ///< NQPT x VDIM x NE (values) / NQPT x VDIM x DIM x NE (grads)
+   byVDIM    ///< VDIM x NQPT x NE (values) / VDIM x DIM x NQPT x NE (grads)
+};
+
 template <> inline int
 Ordering::Map<Ordering::byNODES>(int ndofs, int vdim, int dof, int vd)
 {
@@ -396,7 +404,7 @@ public:
    FiniteElementSpace();
 
    /** @brief Copy constructor: deep copy all data from @a orig except the Mesh,
-       the FiniteElementCollection, ans some derived data. */
+       the FiniteElementCollection, and some derived data. */
    /** If the @a mesh or @a fec pointers are NULL (default), then the new
        FiniteElementSpace will reuse the respective pointers from @a orig. If
        any of these pointers is not NULL, the given pointer will be used instead
@@ -508,7 +516,8 @@ public:
        L2ElementRestriction class.
 
        The returned Operator is owned by the FiniteElementSpace. */
-   const Operator *GetElementRestriction(ElementDofOrdering e_ordering) const;
+   const ElementRestrictionOperator *GetElementRestriction(
+      ElementDofOrdering e_ordering) const;
 
    /// Return an Operator that converts L-vectors to E-vectors on each face.
    virtual const FaceRestriction *GetFaceRestriction(
@@ -929,57 +938,6 @@ public:
    virtual ~FiniteElementSpace();
 };
 
-
-/// Class representing the storage layout of a QuadratureFunction.
-/** Multiple QuadratureFunction%s can share the same QuadratureSpace. */
-class QuadratureSpace
-{
-protected:
-   friend class QuadratureFunction; // Uses the element_offsets.
-
-   Mesh *mesh;
-   int order;
-   int size;
-
-   const IntegrationRule *int_rule[Geometry::NumGeom];
-   int *element_offsets; // scalar offsets; size = number of elements + 1
-
-   // protected functions
-
-   // Assuming mesh and order are set, construct the members: int_rule,
-   // element_offsets, and size.
-   void Construct();
-
-public:
-   /// Create a QuadratureSpace based on the global rules from #IntRules.
-   QuadratureSpace(Mesh *mesh_, int order_)
-      : mesh(mesh_), order(order_) { Construct(); }
-
-   /// Read a QuadratureSpace from the stream @a in.
-   QuadratureSpace(Mesh *mesh_, std::istream &in);
-
-   virtual ~QuadratureSpace() { delete [] element_offsets; }
-
-   /// Return the total number of quadrature points.
-   int GetSize() const { return size; }
-
-   /// Return the order of the quadrature rule(s) used by all elements.
-   int GetOrder() const { return order; }
-
-   /// Returns the mesh
-   inline Mesh *GetMesh() const { return mesh; }
-
-   /// Returns number of elements in the mesh.
-   inline int GetNE() const { return mesh->GetNE(); }
-
-   /// Get the IntegrationRule associated with mesh element @a idx.
-   const IntegrationRule &GetElementIntRule(int idx) const
-   { return *int_rule[mesh->GetElementBaseGeometry(idx)]; }
-
-   /// Write the QuadratureSpace to the stream @a out.
-   void Save(std::ostream &out) const;
-};
-
 /// @brief Return true if the mesh contains only one topology and the elements are tensor elements.
 inline bool UsesTensorBasis(const FiniteElementSpace& fes)
 {

fem/gridfunc.cpp

@@ -12,6 +12,7 @@
 // Implementation of GridFunction
 
 #include "gridfunc.hpp"
+#include "quadinterpolator.hpp"
 #include "../mesh/nurbs.hpp"
 #include "../general/text.hpp"
 
@@ -188,6 +189,8 @@ void GridFunction::Update()
    {
       SetSize(fes->GetVSize());
    }
+
+   if (t_vec.Size() > 0) { SetTrueVector(); }
 }
 
 void GridFunction::SetSpace(FiniteElementSpace *f)
@@ -2761,7 +2764,8 @@ void GridFunction::ProjectBdrCoefficientTangent(
 }
 
 double GridFunction::ComputeL2Error(
-   Coefficient *exsol[], const IntegrationRule *irs[]) const
+   Coefficient *exsol[], const IntegrationRule *irs[],
+   const Array<int> *elems) const
 {
    double error = 0.0, a;
    const FiniteElement *fe;
@@ -2772,6 +2776,7 @@ double GridFunction::ComputeL2Error(
 
    for (i = 0; i < fes->GetNE(); i++)
    {
+      if (elems != NULL && (*elems)[i] == 0) { continue; }
       fe = fes->GetFE(i);
       fdof = fe->GetDof();
       transf = fes->GetElementTransformation(i);
@@ -2815,7 +2820,7 @@ double GridFunction::ComputeL2Error(
 
 double GridFunction::ComputeL2Error(
    VectorCoefficient &exsol, const IntegrationRule *irs[],
-   Array<int> *elems) const
+   const Array<int> *elems) const
 {
    double error = 0.0;
    const FiniteElement *fe;
@@ -3234,7 +3239,7 @@ double GridFunction::ComputeMaxError(
 
 double GridFunction::ComputeW11Error(
    Coefficient *exsol, VectorCoefficient *exgrad, int norm_type,
-   Array<int> *elems, const IntegrationRule *irs[]) const
+   const Array<int> *elems, const IntegrationRule *irs[]) const
 {
    // assuming vdim is 1
    int i, fdof, dim, intorder, j, k;
@@ -3340,7 +3345,8 @@ double GridFunction::ComputeW11Error(
 
 double GridFunction::ComputeLpError(const double p, Coefficient &exsol,
                                     Coefficient *weight,
-                                    const IntegrationRule *irs[]) const
+                                    const IntegrationRule *irs[],
+                                    const Array<int> *elems) const
 {
    double error = 0.0;
    const FiniteElement *fe;
@@ -3349,6 +3355,7 @@ double GridFunction::ComputeLpError(const double p, Coefficient &exsol,
 
    for (int i = 0; i < fes->GetNE(); i++)
    {
+      if (elems != NULL && (*elems)[i] == 0) { continue; }
       fe = fes->GetFE(i);
       const IntegrationRule *ir;
       if (irs)
@@ -3968,178 +3975,6 @@ void GridFunction::LegacyNCReorder()
    Vector::Swap(tmp);
 }
 
-
-QuadratureFunction::QuadratureFunction(Mesh *mesh, std::istream &in)
-{
-   const char *msg = "invalid input stream";
-   string ident;
-
-   qspace = new QuadratureSpace(mesh, in);
-   own_qspace = true;
-
-   in >> ident; MFEM_VERIFY(ident == "VDim:", msg);
-   in >> vdim;
-
-   Load(in, vdim*qspace->GetSize());
-}
-
-QuadratureFunction & QuadratureFunction::operator=(double value)
-{
-   Vector::operator=(value);
-   return *this;
-}
-
-QuadratureFunction & QuadratureFunction::operator=(const Vector &v)
-{
-   MFEM_ASSERT(qspace && v.Size() == this->Size(), "");
-   Vector::operator=(v);
-   return *this;
-}
-
-QuadratureFunction & QuadratureFunction::operator=(const QuadratureFunction &v)
-{
-   return this->operator=((const Vector &)v);
-}
-
-void QuadratureFunction::Save(std::ostream &os) const
-{
-   qspace->Save(os);
-   os << "VDim: " << vdim << '\n'
-      << '\n';
-   Vector::Print(os, vdim);
-   os.flush();
-}
-
-std::ostream &operator<<(std::ostream &os, const QuadratureFunction &qf)
-{
-   qf.Save(os);
-   return os;
-}
-
-void QuadratureFunction::SaveVTU(std::ostream &os, VTKFormat format,
-                                 int compression_level) const
-{
-   os << R"(<VTKFile type="UnstructuredGrid" version="0.1")";
-   if (compression_level != 0)
-   {
-      os << R"( compressor="vtkZLibDataCompressor")";
-   }
-   os << " byte_order=\"" << VTKByteOrder() << "\">\n";
-   os << "<UnstructuredGrid>\n";
-
-   const char *fmt_str = (format == VTKFormat::ASCII) ? "ascii" : "binary";
-   const char *type_str = (format != VTKFormat::BINARY32) ? "Float64" : "Float32";
-   std::vector<char> buf;
-
-   int np = qspace->GetSize();
-   int ne = qspace->GetNE();
-   int sdim = qspace->GetMesh()->SpaceDimension();
-
-   // For quadrature functions, each point is a vertex cell, so number of cells
-   // is equal to number of points
-   os << "<Piece NumberOfPoints=\"" << np
-      << "\" NumberOfCells=\"" << np << "\">\n";
-
-   // print out the points
-   os << "<Points>\n";
-   os << "<DataArray type=\"" << type_str
-      << "\" NumberOfComponents=\"3\" format=\"" << fmt_str << "\">\n";
-
-   Vector pt(sdim);
-   for (int i = 0; i < ne; i++)
-   {
-      ElementTransformation &T = *qspace->GetMesh()->GetElementTransformation(i);
-      const IntegrationRule &ir = GetElementIntRule(i);
-      for (int j = 0; j < ir.Size(); j++)
-      {
-         T.Transform(ir[j], pt);
-         WriteBinaryOrASCII(os, buf, pt[0], " ", format);
-         if (sdim > 1) { WriteBinaryOrASCII(os, buf, pt[1], " ", format); }
-         else { WriteBinaryOrASCII(os, buf, 0.0, " ", format); }
-         if (sdim > 2) { WriteBinaryOrASCII(os, buf, pt[2], "", format); }
-         else { WriteBinaryOrASCII(os, buf, 0.0, "", format); }
-         if (format == VTKFormat::ASCII) { os << '\n'; }
-      }
-   }
-   if (format != VTKFormat::ASCII)
-   {
-      WriteBase64WithSizeAndClear(os, buf, compression_level);
-   }
-   os << "</DataArray>\n";
-   os << "</Points>\n";
-
-   // Write cells (each cell is just a vertex)
-   os << "<Cells>\n";
-   // Connectivity
-   os << R"(<DataArray type="Int32" Name="connectivity" format=")"
-      << fmt_str << "\">\n";
-
-   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(os, buf, i, "\n", format); }
-   if (format != VTKFormat::ASCII)
-   {
-      WriteBase64WithSizeAndClear(os, buf, compression_level);
-   }
-   os << "</DataArray>\n";
-   // Offsets
-   os << R"(<DataArray type="Int32" Name="offsets" format=")"
-      << fmt_str << "\">\n";
-   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(os, buf, i, "\n", format); }
-   if (format != VTKFormat::ASCII)
-   {
-      WriteBase64WithSizeAndClear(os, buf, compression_level);
-   }
-   os << "</DataArray>\n";
-   // Types
-   os << R"(<DataArray type="UInt8" Name="types" format=")"
-      << fmt_str << "\">\n";
-   for (int i = 0; i < np; i++)
-   {
-      uint8_t vtk_cell_type = VTKGeometry::POINT;
-      WriteBinaryOrASCII(os, buf, vtk_cell_type, "\n", format);
-   }
-   if (format != VTKFormat::ASCII)
-   {
-      WriteBase64WithSizeAndClear(os, buf, compression_level);
-   }
-   os << "</DataArray>\n";
-   os << "</Cells>\n";
-
-   os << "<PointData>\n";
-   os << "<DataArray type=\"" << type_str << "\" Name=\"u\" format=\""
-      << fmt_str << "\" NumberOfComponents=\"" << vdim << "\">\n";
-   for (int i = 0; i < ne; i++)
-   {
-      DenseMatrix vals;
-      GetElementValues(i, vals);
-      for (int j = 0; j < vals.Size(); ++j)
-      {
-         for (int vd = 0; vd < vdim; ++vd)
-         {
-            WriteBinaryOrASCII(os, buf, vals(vd, j), " ", format);
-         }
-         if (format == VTKFormat::ASCII) { os << '\n'; }
-      }
-   }
-   if (format != VTKFormat::ASCII)
-   {
-      WriteBase64WithSizeAndClear(os, buf, compression_level);
-   }
-   os << "</DataArray>\n";
-   os << "</PointData>\n";
-
-   os << "</Piece>\n";
-   os << "</UnstructuredGrid>\n";
-   os << "</VTKFile>" << std::endl;
-}
-
-void QuadratureFunction::SaveVTU(const std::string &filename, VTKFormat format,
-                                 int compression_level) const
-{
-   std::ofstream f(filename + ".vtu");
-   SaveVTU(f, format, compression_level);
-}
-
-
 double ZZErrorEstimator(BilinearFormIntegrator &blfi,
                         GridFunction &u,
                         GridFunction &flux, Vector &error_estimates,
@@ -4281,44 +4116,44 @@ void TensorProductLegendre(int dim,                // input
    switch (dim)
    {
       case 1:
+      {
+         for (int i = 0; i <= order; i++)
+         {
+            poly(i) = poly_x(i);
+         }
+      }
+      break;
+      case 2:
+      {
+         for (int j = 0; j <= order; j++)
          {
             for (int i = 0; i <= order; i++)
             {
-               poly(i) = poly_x(i);
+               int cnt = i + (order+1) * j;
+               poly(cnt) = poly_x(i) * poly_y(j);
             }
          }
-         break;
-      case 2:
+      }
+      break;
+      case 3:
+      {
+         for (int k = 0; k <= order; k++)
          {
             for (int j = 0; j <= order; j++)
             {
                for (int i = 0; i <= order; i++)
                {
-                  int cnt = i + (order+1) * j;
-                  poly(cnt) = poly_x(i) * poly_y(j);
+                  int cnt = i + (order+1) * j + (order+1) * (order+1) * k;
+                  poly(cnt) = poly_x(i) * poly_y(j) * poly_z(k);
                }
             }
          }
-         break;
-      case 3:
-         {
-            for (int k = 0; k <= order; k++)
-            {
-               for (int j = 0; j <= order; j++)
-               {
-                  for (int i = 0; i <= order; i++)
-                  {
-                     int cnt = i + (order+1) * j + (order+1) * (order+1) * k;
-                     poly(cnt) = poly_x(i) * poly_y(j) * poly_z(k);
-                  }
-               }
-            }
-         }
-         break;
+      }
+      break;
       default:
-         {
-            MFEM_ABORT("TensorProductLegendre: invalid value of dim");
-         }
+      {
+         MFEM_ABORT("TensorProductLegendre: invalid value of dim");
+      }
    }
 }
 

fem/gridfunc.hpp

@@ -129,19 +129,21 @@ public:
    int CurlDim() const;
 
    /// Read only access to the (optional) internal true-dof Vector.
-   /** Note that the returned Vector may be empty, if not previously allocated
-       or set. */
-   const Vector &GetTrueVector() const { return t_vec; }
+   const Vector &GetTrueVector() const
+   {
+      MFEM_VERIFY(t_vec.Size() > 0, "SetTrueVector() before GetTrueVector()");
+      return t_vec;
+   }
    /// Read and write access to the (optional) internal true-dof Vector.
-   /** Note that the returned Vector may be empty, if not previously allocated
-       or set. */
-   Vector &GetTrueVector() { return t_vec; }
+   /** Note that @a t_vec is set if it is not allocated or set already.*/
+   Vector &GetTrueVector()
+   { if (t_vec.Size() == 0) { SetTrueVector(); } return t_vec; }
 
    /// Extract the true-dofs from the GridFunction.
    void GetTrueDofs(Vector &tv) const;
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.
-   void SetTrueVector() { GetTrueDofs(GetTrueVector()); }
+   void SetTrueVector() { GetTrueDofs(t_vec); }
 
    /// Set the GridFunction from the given true-dof vector.
    virtual void SetFromTrueDofs(const Vector &tv);
@@ -476,21 +478,27 @@ public:
                                              Array<int> &bdr_attr);
 
 
-   virtual double ComputeL2Error(Coefficient &exsol,
-                                 const IntegrationRule *irs[] = NULL) const
-   { return ComputeLpError(2.0, exsol, NULL, irs); }
-
    virtual double ComputeL2Error(Coefficient *exsol[],
-                                 const IntegrationRule *irs[] = NULL) const;
-
-   virtual double ComputeL2Error(VectorCoefficient &exsol,
                                  const IntegrationRule *irs[] = NULL,
-                                 Array<int> *elems = NULL) const;
+                                 const Array<int> *elems = NULL) const;
 
    /// Returns ||grad u_ex - grad u_h||_L2 in element ielem for H1 or L2 elements
    virtual double ComputeElementGradError(int ielem, VectorCoefficient *exgrad,
                                           const IntegrationRule *irs[] = NULL) const;
 
+   /// Returns ||u_ex - u_h||_L2 for H1 or L2 elements
+   /* The @a elems input variable expects a list of markers:
+      an elem marker equal to 1 will compute the L2 error on that element
+      an elem marker equal to 0 will not compute the L2 error on that element */
+   virtual double ComputeL2Error(Coefficient &exsol,
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const
+   { return GridFunction::ComputeLpError(2.0, exsol, NULL, irs, elems); }
+
+   virtual double ComputeL2Error(VectorCoefficient &exsol,
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const;
+
    /// Returns ||grad u_ex - grad u_h||_L2 for H1 or L2 elements
    virtual double ComputeGradError(VectorCoefficient *exgrad,
                                    const IntegrationRule *irs[] = NULL) const;
@@ -564,16 +572,20 @@ public:
    { return ComputeLpError(1.0, exsol, NULL, irs); }
 
    virtual double ComputeW11Error(Coefficient *exsol, VectorCoefficient *exgrad,
-                                  int norm_type, Array<int> *elems = NULL,
+                                  int norm_type, const Array<int> *elems = NULL,
                                   const IntegrationRule *irs[] = NULL) const;
 
    virtual double ComputeL1Error(VectorCoefficient &exsol,
                                  const IntegrationRule *irs[] = NULL) const
    { return ComputeLpError(1.0, exsol, NULL, NULL, irs); }
 
+   /* The @a elems input variable expects a list of markers:
+    an elem marker equal to 1 will compute the L2 error on that element
+    an elem marker equal to 0 will not compute the L2 error on that element */
    virtual double ComputeLpError(const double p, Coefficient &exsol,
                                  Coefficient *weight = NULL,
-                                 const IntegrationRule *irs[] = NULL) const;
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const;
 
    /** Compute the Lp error in each element of the mesh and store the results in
        the Vector @a error. The result should be of length number of elements,
@@ -755,176 +767,6 @@ public:
    }
 };
 
-
-/** @brief Class representing a function through its values (scalar or vector)
-    at quadrature points. */
-class QuadratureFunction : public Vector
-{
-protected:
-   QuadratureSpace *qspace; ///< Associated QuadratureSpace
-   int vdim;                ///< Vector dimension
-   bool own_qspace;         ///< QuadratureSpace ownership flag
-
-public:
-   /// Create an empty QuadratureFunction.
-   /** The object can be initialized later using the SetSpace() methods. */
-   QuadratureFunction()
-      : qspace(NULL), vdim(0), own_qspace(false) { }
-
-   /** @brief Copy constructor. The QuadratureSpace ownership flag, #own_qspace,
-       in the new object is set to false. */
-   QuadratureFunction(const QuadratureFunction &orig)
-      : Vector(orig),
-        qspace(orig.qspace), vdim(orig.vdim), own_qspace(false) { }
-
-   /// Create a QuadratureFunction based on the given QuadratureSpace.
-   /** The QuadratureFunction does not assume ownership of the QuadratureSpace.
-       @note The Vector data is not initialized. */
-   QuadratureFunction(QuadratureSpace *qspace_, int vdim_ = 1)
-      : Vector(vdim_*qspace_->GetSize()),
-        qspace(qspace_), vdim(vdim_), own_qspace(false) { }
-
-   /** @brief Create a QuadratureFunction based on the given QuadratureSpace,
-       using the external data, @a qf_data. */
-   /** The QuadratureFunction does not assume ownership of neither the
-       QuadratureSpace nor the external data. */
-   QuadratureFunction(QuadratureSpace *qspace_, double *qf_data, int vdim_ = 1)
-      : Vector(qf_data, vdim_*qspace_->GetSize()),
-        qspace(qspace_), vdim(vdim_), own_qspace(false) { }
-
-   /// Read a QuadratureFunction from the stream @a in.
-   /** The QuadratureFunction assumes ownership of the read QuadratureSpace. */
-   QuadratureFunction(Mesh *mesh, std::istream &in);
-
-   virtual ~QuadratureFunction() { if (own_qspace) { delete qspace; } }
-
-   /// Get the associated QuadratureSpace.
-   QuadratureSpace *GetSpace() const { return qspace; }
-
-   /// Change the QuadratureSpace and optionally the vector dimension.
-   /** If the new QuadratureSpace is different from the current one, the
-       QuadratureFunction will not assume ownership of the new space; otherwise,
-       the ownership flag remains the same.
-
-       If the new vector dimension @a vdim_ < 0, the vector dimension remains
-       the same.
-
-       The data size is updated by calling Vector::SetSize(). */
-   inline void SetSpace(QuadratureSpace *qspace_, int vdim_ = -1);
-
-   /** @brief Change the QuadratureSpace, the data array, and optionally the
-       vector dimension. */
-   /** If the new QuadratureSpace is different from the current one, the
-       QuadratureFunction will not assume ownership of the new space; otherwise,
-       the ownership flag remains the same.
-
-       If the new vector dimension @a vdim_ < 0, the vector dimension remains
-       the same.
-
-       The data array is replaced by calling Vector::NewDataAndSize(). */
-   inline void SetSpace(QuadratureSpace *qspace_, double *qf_data,
-                        int vdim_ = -1);
-
-   /// Get the vector dimension.
-   int GetVDim() const { return vdim; }
-
-   /// Set the vector dimension, updating the size by calling Vector::SetSize().
-   void SetVDim(int vdim_)
-   { vdim = vdim_; SetSize(vdim*qspace->GetSize()); }
-
-   /// Get the QuadratureSpace ownership flag.
-   bool OwnsSpace() { return own_qspace; }
-
-   /// Set the QuadratureSpace ownership flag.
-   void SetOwnsSpace(bool own) { own_qspace = own; }
-
-   /// Redefine '=' for QuadratureFunction = constant.
-   QuadratureFunction &operator=(double value);
-
-   /// Copy the data from @a v.
-   /** The size of @a v must be equal to the size of the associated
-       QuadratureSpace #qspace times the QuadratureFunction dimension
-       i.e. QuadratureFunction::Size(). */
-   QuadratureFunction &operator=(const Vector &v);
-
-   /// Copy assignment. Only the data of the base class Vector is copied.
-   /** The QuadratureFunctions @a v and @a *this must have QuadratureSpaces with
-       the same size.
-
-       @note Defining this method overwrites the implicitly defined copy
-       assignment operator. */
-   QuadratureFunction &operator=(const QuadratureFunction &v);
-
-   /// Get the IntegrationRule associated with mesh element @a idx.
-   const IntegrationRule &GetElementIntRule(int idx) const
-   { return qspace->GetElementIntRule(idx); }
-
-   /// Return all values associated with mesh element @a idx in a Vector.
-   /** The result is stored in the Vector @a values as a reference to the
-       global values.
-
-       Inside the Vector @a values, the index `i+vdim*j` corresponds to the
-       `i`-th vector component at the `j`-th quadrature point.
-    */
-   inline void GetElementValues(int idx, Vector &values);
-
-   /// Return all values associated with mesh element @a idx in a Vector.
-   /** The result is stored in the Vector @a values as a copy of the
-       global values.
-
-       Inside the Vector @a values, the index `i+vdim*j` corresponds to the
-       `i`-th vector component at the `j`-th quadrature point.
-    */
-   inline void GetElementValues(int idx, Vector &values) const;
-
-   /// Return the quadrature function values at an integration point.
-   /** The result is stored in the Vector @a values as a reference to the
-       global values. */
-   inline void GetElementValues(int idx, const int ip_num, Vector &values);
-
-   /// Return the quadrature function values at an integration point.
-   /** The result is stored in the Vector @a values as a copy to the
-       global values. */
-   inline void GetElementValues(int idx, const int ip_num, Vector &values) const;
-
-   /// Return all values associated with mesh element @a idx in a DenseMatrix.
-   /** The result is stored in the DenseMatrix @a values as a reference to the
-       global values.
-
-       Inside the DenseMatrix @a values, the `(i,j)` entry corresponds to the
-       `i`-th vector component at the `j`-th quadrature point.
-    */
-   inline void GetElementValues(int idx, DenseMatrix &values);
-
-   /// Return all values associated with mesh element @a idx in a const DenseMatrix.
-   /** The result is stored in the DenseMatrix @a values as a copy of the
-       global values.
-
-       Inside the DenseMatrix @a values, the `(i,j)` entry corresponds to the
-       `i`-th vector component at the `j`-th quadrature point.
-    */
-   inline void GetElementValues(int idx, DenseMatrix &values) const;
-
-   /// Write the QuadratureFunction to the stream @a out.
-   void Save(std::ostream &out) const;
-
-   /// @brief Write the QuadratureFunction to @a out in VTU (ParaView) format.
-   ///
-   /// The data will be uncompressed if @a compression_level is zero, or if the
-   /// format is VTKFormat::ASCII. Otherwise, zlib compression will be used for
-   /// binary data.
-   void SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
-                int compression_level=0) const;
-
-   /// @brief Save the QuadratureFunction to a VTU (ParaView) file.
-   ///
-   /// The extension ".vtu" will be appended to @a filename.
-   /// @sa SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
-   ///             int compression_level=0)
-   void SaveVTU(const std::string &filename, VTKFormat format=VTKFormat::ASCII,
-                int compression_level=0) const;
-};
-
 /// Overload operator<< for std::ostream and QuadratureFunction.
 std::ostream &operator<<(std::ostream &out, const QuadratureFunction &qf);
 
@@ -1012,95 +854,6 @@ public:
 GridFunction *Extrude1DGridFunction(Mesh *mesh, Mesh *mesh2d,
                                     GridFunction *sol, const int ny);
 
-
-// Inline methods
-
-inline void QuadratureFunction::SetSpace(QuadratureSpace *qspace_, int vdim_)
-{
-   if (qspace_ != qspace)
-   {
-      if (own_qspace) { delete qspace; }
-      qspace = qspace_;
-      own_qspace = false;
-   }
-   vdim = (vdim_ < 0) ? vdim : vdim_;
-   SetSize(vdim*qspace->GetSize());
-}
-
-inline void QuadratureFunction::SetSpace(QuadratureSpace *qspace_,
-                                         double *qf_data, int vdim_)
-{
-   if (qspace_ != qspace)
-   {
-      if (own_qspace) { delete qspace; }
-      qspace = qspace_;
-      own_qspace = false;
-   }
-   vdim = (vdim_ < 0) ? vdim : vdim_;
-   NewDataAndSize(qf_data, vdim*qspace->GetSize());
-}
-
-inline void QuadratureFunction::GetElementValues(int idx, Vector &values)
-{
-   const int s_offset = qspace->element_offsets[idx];
-   const int sl_size = qspace->element_offsets[idx+1] - s_offset;
-   values.NewDataAndSize(data + vdim*s_offset, vdim*sl_size);
-}
-
-inline void QuadratureFunction::GetElementValues(int idx, Vector &values) const
-{
-   const int s_offset = qspace->element_offsets[idx];
-   const int sl_size = qspace->element_offsets[idx+1] - s_offset;
-   values.SetSize(vdim*sl_size);
-   const double *q = data + vdim*s_offset;
-   for (int i = 0; i<values.Size(); i++)
-   {
-      values(i) = *(q++);
-   }
-}
-
-inline void QuadratureFunction::GetElementValues(int idx, const int ip_num,
-                                                 Vector &values)
-{
-   const int s_offset = qspace->element_offsets[idx] * vdim + ip_num * vdim;
-   values.NewDataAndSize(data + s_offset, vdim);
-}
-
-inline void QuadratureFunction::GetElementValues(int idx, const int ip_num,
-                                                 Vector &values) const
-{
-   const int s_offset = qspace->element_offsets[idx] * vdim + ip_num * vdim;
-   values.SetSize(vdim);
-   const double *q = data + s_offset;
-   for (int i = 0; i < values.Size(); i++)
-   {
-      values(i) = *(q++);
-   }
-}
-
-inline void QuadratureFunction::GetElementValues(int idx, DenseMatrix &values)
-{
-   const int s_offset = qspace->element_offsets[idx];
-   const int sl_size = qspace->element_offsets[idx+1] - s_offset;
-   values.Reset(data + vdim*s_offset, vdim, sl_size);
-}
-
-inline void QuadratureFunction::GetElementValues(int idx,
-                                                 DenseMatrix &values) const
-{
-   const int s_offset = qspace->element_offsets[idx];
-   const int sl_size = qspace->element_offsets[idx+1] - s_offset;
-   values.SetSize(vdim, sl_size);
-   const double *q = data + vdim*s_offset;
-   for (int j = 0; j<sl_size; j++)
-   {
-      for (int i = 0; i<vdim; i++)
-      {
-         values(i,j) = *(q++);
-      }
-   }
-}
-
 } // namespace mfem
 
 #endif

fem/kernels.hpp

@@ -120,6 +120,22 @@ MFEM_HOST_DEVICE inline void LoadBGt(const int D1D, const int Q1D,
    MFEM_SYNC_THREAD;
 }
 
+/// Load 2D input scalar into given DeviceMatrix
+MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
+                                   const DeviceTensor<3, const double> &x,
+                                   DeviceMatrix &DD)
+{
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         DD(dx,dy) = x(dx,dy,e);
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+
 /// Load 2D input scalar into shared memory
 template<int MD1, int NBZ>
 MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
@@ -128,15 +144,7 @@ MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
 {
    const int tidz = MFEM_THREAD_ID(z);
    DeviceMatrix X(sX[tidz], D1D, D1D);
-
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         X(dx,dy) = x(dx,dy,e);
-      }
-   }
-   MFEM_SYNC_THREAD;
+   LoadX(e, D1D, x, X);
 }
 
 /// Load 2D input scalar into shared memory, with comp

fem/linearform.cpp

@@ -106,19 +106,40 @@ bool LinearForm::SupportsDevice()
    // through Assemble, AssembleDevice, GetGeometricFactors and EnsureNodes
    if (fes->GetMesh()->NURBSext != nullptr) { return false; }
 
-   // scan domain integrator to verify that all can use device assembly
-   if (domain_integs.Size() > 0)
+   // scan integrators to verify that all can use device assembly
+   auto IntegratorsSupportDevice = [](const Array<LinearFormIntegrator*> &integ)
    {
-      for (int k = 0; k < domain_integs.Size(); k++)
+      for (int k = 0; k < integ.Size(); k++)
       {
-         if (!domain_integs[k]->SupportsDevice()) { return false; }
+         if (!integ[k]->SupportsDevice()) { return false; }
+      }
+      return true;
+   };
+
+   if (!IntegratorsSupportDevice(domain_integs)) { return false; }
+   if (!IntegratorsSupportDevice(boundary_integs)) { return false; }
+   if (boundary_face_integs.Size() > 0 || interior_face_integs.Size() > 0 ||
+       domain_delta_integs.Size() > 0) { return false; }
+
+   if (boundary_integs.Size() > 0)
+   {
+      // Make sure every boundary element corresponds to a boundary face
+      for (int be = 0; be < fes->GetNBE(); ++be)
+      {
+         const int f = fes->GetMesh()->GetBdrElementEdgeIndex(be);
+         const auto face_info = fes->GetMesh()->GetFaceInformation(f);
+         if (!face_info.IsBoundary())
+         {
+            return false;
+         }
+      }
+      // Make sure there are no boundary faces that are not boundary elements
+      if (fes->GetNFbyType(FaceType::Boundary) != fes->GetNBE())
+      {
+         return false;
       }
    }
 
-   // boundary, delta and face integrators are not supported yet
-   if (GetBLFI()->Size() > 0 || GetFLFI()->Size() > 0 ||
-       GetDLFI_Delta()->Size() > 0 || GetIFLFI()->Size() > 0) { return false; }
-
    const Mesh &mesh = *fes->GetMesh();
 
    // no support for elements with varying polynomial orders
@@ -173,8 +194,8 @@ void LinearForm::Assemble(bool use_device)
          int elem_attr = fes->GetMesh()->GetAttribute(i);
          for (int k = 0; k < domain_integs.Size(); k++)
          {
-            if ( domain_integs_marker[k] == NULL ||
-                 (*(domain_integs_marker[k]))[elem_attr-1] == 1 )
+            const Array<int> * const markers = domain_integs_marker[k];
+            if ( markers == NULL || (*markers)[elem_attr-1] == 1 )
             {
                doftrans = fes -> GetElementVDofs (i, vdofs);
                eltrans = fes -> GetElementTransformation (i);

fem/linearform_ext.cpp

@@ -25,7 +25,8 @@ void LinearFormExtension::Assemble()
                "match the number of vector dofs!");
 
    const Array<Array<int>*> &domain_integs_marker = *lf->GetDLFI_Marker();
-   const int mesh_attributes_size = fes.GetMesh()->attributes.Size();
+   const int mesh_attributes_max = fes.GetMesh()->attributes.Size() ?
+                                   fes.GetMesh()->attributes.Max() : 0;
    const Array<LinearFormIntegrator*> &domain_integs = *lf->GetDLFI();
 
    for (int k = 0; k < domain_integs.Size(); ++k)
@@ -39,7 +40,7 @@ void LinearFormExtension::Assemble()
       if (has_markers_k)
       {
          // Element attribute marker should be of length mesh->attributes
-         MFEM_VERIFY(mesh_attributes_size == domain_integs_marker_k->Size(),
+         MFEM_VERIFY(mesh_attributes_max == domain_integs_marker_k->Size(),
                      "invalid element marker for domain linear form "
                      "integrator #" << k << ", counting from zero");
       }
@@ -59,7 +60,47 @@ void LinearFormExtension::Assemble()
       // Assemble the linear form
       b = 0.0;
       domain_integs[k]->AssembleDevice(fes, markers, b);
-      elem_restrict_lex->MultTranspose(b, *lf);
+      if (k == 0) { elem_restrict_lex->MultTranspose(b, *lf); }
+      else { elem_restrict_lex->AddMultTranspose(b, *lf); }
+   }
+
+   const Array<Array<int>*> &boundary_integs_marker = lf->boundary_integs_marker;
+   const int bdr_attributes_max = fes.GetMesh()->bdr_attributes.Size() ?
+                                  fes.GetMesh()->bdr_attributes.Max() : 0;
+   const Array<LinearFormIntegrator*> &boundary_integs = lf->boundary_integs;
+
+   for (int k = 0; k < boundary_integs.Size(); ++k)
+   {
+      // Get the markers for this integrator
+      const Array<int> *boundary_integs_marker_k = boundary_integs_marker[k];
+
+      // check if there are markers for this integrator
+      const bool has_markers_k = boundary_integs_marker_k != nullptr;
+
+      if (has_markers_k)
+      {
+         // Element attribute marker should be of length mesh->attributes
+         MFEM_VERIFY(bdr_attributes_max == boundary_integs_marker_k->Size(),
+                     "invalid boundary marker for boundary linear form "
+                     "integrator #" << k << ", counting from zero");
+      }
+
+      // if there are no markers, just use the whole linear form (1)
+      if (!has_markers_k) { bdr_markers.HostReadWrite(); bdr_markers = 1; }
+      else
+      {
+         // scan the attributes to set the markers to 0 or 1
+         const int NBE = bdr_attributes.Size();
+         const auto attr = bdr_attributes.Read();
+         const auto attr_markers = boundary_integs_marker_k->Read();
+         auto markers_w = bdr_markers.Write();
+         MFEM_FORALL(e, NBE, markers_w[e] = attr_markers[attr[e]-1] == 1;);
+      }
+
+      // Assemble the linear form
+      bdr_b = 0.0;
+      boundary_integs[k]->AssembleDevice(fes, bdr_markers, bdr_b);
+      bdr_restrict_lex->AddMultTranspose(bdr_b, *lf);
    }
 }
 
@@ -67,22 +108,64 @@ void LinearFormExtension::Update()
 {
    const FiniteElementSpace &fes = *lf->FESpace();
    const Mesh &mesh = *fes.GetMesh();
-   const int NE = fes.GetNE();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
 
    MFEM_VERIFY(lf->Size() == fes.GetVSize(), "");
 
-   markers.SetSize(NE);
-   //markers.UseDevice(true);
+   if (lf->domain_integs.Size() > 0)
+   {
+      const int NE = fes.GetNE();
+      markers.SetSize(NE);
+      //markers.UseDevice(true);
 
-   // Gather the attributes on the host from all the elements
-   attributes.SetSize(NE);
-   for (int i = 0; i < NE; ++i) { attributes[i] = mesh.GetAttribute(i); }
+      // Gather the attributes on the host from all the elements
+      attributes.SetSize(NE);
+      for (int i = 0; i < NE; ++i) { attributes[i] = mesh.GetAttribute(i); }
 
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   elem_restrict_lex = fes.GetElementRestriction(ordering);
-   MFEM_VERIFY(elem_restrict_lex, "Element restriction not available");
-   b.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-   b.UseDevice(true);
+      elem_restrict_lex = fes.GetElementRestriction(ordering);
+      MFEM_VERIFY(elem_restrict_lex, "Element restriction not available");
+      b.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
+      b.UseDevice(true);
+   }
+
+   if (lf->boundary_integs.Size() > 0)
+   {
+      const int nf_bdr = fes.GetNFbyType(FaceType::Boundary);
+      bdr_markers.SetSize(nf_bdr);
+      // bdr_markers.UseDevice(true);
+
+      // The face restriction will give us "face E-vectors" on the boundary that
+      // are numbered in the order of the faces of mesh. This numbering will be
+      // different than the numbering of the boundary elements. We compute
+      // mappings so that the array `bdr_attributes[i]` gives the boundary
+      // attribute of the `i`th boundary face in the mesh face order.
+      std::unordered_map<int,int> f_to_be;
+      for (int i = 0; i < mesh.GetNBE(); ++i)
+      {
+         const int f = mesh.GetBdrElementEdgeIndex(i);
+         f_to_be[f] = i;
+      }
+      MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
+      bdr_attributes.SetSize(nf_bdr);
+      int f_ind = 0;
+      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      {
+         if (f_to_be.find(f) != f_to_be.end())
+         {
+            const int be = f_to_be[f];
+            bdr_attributes[f_ind] = mesh.GetBdrAttribute(be);
+            ++f_ind;
+         }
+      }
+
+      bdr_restrict_lex =
+         dynamic_cast<const FaceRestriction*>(
+            fes.GetFaceRestriction(ordering, FaceType::Boundary,
+                                   L2FaceValues::SingleValued));
+      MFEM_VERIFY(bdr_restrict_lex, "Face restriction not available");
+      bdr_b.SetSize(bdr_restrict_lex->Height(), Device::GetMemoryType());
+      bdr_b.UseDevice(true);
+   }
 }
 
 } // namespace mfem

fem/linearform_ext.hpp

@@ -25,19 +25,22 @@ class LinearForm;
 class LinearFormExtension
 {
    /// Attributes of all mesh elements.
-   Array<int> attributes;
+   Array<int> attributes, bdr_attributes;
 
    /// Temporary markers for device kernels.
-   Array<int> markers;
+   Array<int> markers, bdr_markers;
 
    /// Linear form from which this extension depends. Not owned.
    LinearForm *lf;
 
    /// Operator that converts FiniteElementSpace L-vectors to E-vectors.
-   const Operator *elem_restrict_lex; // Not owned
+   const ElementRestrictionOperator *elem_restrict_lex; // Not owned
+
+   /// Operator that converts L-vectors to boundary E-vectors.
+   const FaceRestriction *bdr_restrict_lex; // Not owned
 
    /// Internal E-vectors.
-   mutable Vector b;
+   mutable Vector b, bdr_b;
 
 public:
 

fem/lininteg.cpp

@@ -1046,7 +1046,7 @@ void VectorQuadratureLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &fe, ElementTransformation &Tr, Vector &elvect)
 {
    const IntegrationRule *ir =
-      &vqfc.GetQuadFunction().GetSpace()->GetElementIntRule(Tr.ElementNo);
+      &vqfc.GetQuadFunction().GetSpace()->GetIntRule(Tr.ElementNo);
 
    const int nqp = ir->GetNPoints();
    const int vdim = vqfc.GetVDim();
@@ -1078,7 +1078,7 @@ void QuadratureLFIntegrator::AssembleRHSElementVect(const FiniteElement &fe,
                                                     Vector &elvect)
 {
    const IntegrationRule *ir =
-      &qfc.GetQuadFunction().GetSpace()->GetElementIntRule(Tr.ElementNo);
+      &qfc.GetQuadFunction().GetSpace()->GetIntRule(Tr.ElementNo);
 
    const int nqp = ir->GetNPoints();
    const int ndofs = fe.GetDof();

fem/lininteg.hpp

@@ -187,6 +187,13 @@ public:
    BoundaryLFIntegrator(Coefficient &QG, int a = 1, int b = 1)
       : Q(QG), oa(a), ob(b) { }
 
+   virtual bool SupportsDevice() { return true; }
+
+   /// Method defining assembly on device
+   virtual void AssembleDevice(const FiniteElementSpace &fes,
+                               const Array<int> &markers,
+                               Vector &b);
+
    /** Given a particular boundary Finite Element and a transformation (Tr)
        computes the element boundary vector, elvect. */
    virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -210,6 +217,13 @@ public:
    BoundaryNormalLFIntegrator(VectorCoefficient &QG, int a = 1, int b = 1)
       : Q(QG), oa(a), ob(b) { }
 
+   virtual bool SupportsDevice() { return true; }
+
+   /// Method defining assembly on device
+   virtual void AssembleDevice(const FiniteElementSpace &fes,
+                               const Array<int> &markers,
+                               Vector &b);
+
    virtual void AssembleRHSElementVect(const FiniteElement &el,
                                        ElementTransformation &Tr,
                                        Vector &elvect);

fem/lininteg_boundary.cpp

@@ -0,0 +1,241 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "../fem/kernels.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+template<int T_D1D = 0, int T_Q1D = 0> static
+void BLFEvalAssemble2D(const int vdim, const int nbe, const int d, const int q,
+                       const bool normals, const int *markers, const double *b,
+                       const double *detj, const double *n, const double *weights,
+                       const Vector &coeff, double *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, nbe);
+   const auto B = Reshape(b, q, d);
+   const auto detJ = Reshape(detj, q, nbe);
+   const auto N = Reshape(n, q, 2, nbe);
+   const auto W = Reshape(weights, q);
+   const int cvdim = normals ? 2 : 1;
+   const bool cst = coeff.Size() == cvdim;
+   const auto C = cst ? Reshape(F,cvdim,1,1) : Reshape(F,cvdim,q,nbe);
+   auto Y = Reshape(y, d, vdim, nbe);
+
+   MFEM_FORALL(e, nbe,
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQ[Q];
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         for (int qx = 0; qx < q; ++qx)
+         {
+            double coeff_val = 0.0;
+            if (normals)
+            {
+               for (int cd = 0; cd < 2; ++cd)
+               {
+                  const double cval = cst ? C(cd,0,0) : C(cd,qx,e);
+                  coeff_val += cval * N(qx, cd, e);
+               }
+            }
+            else
+            {
+               coeff_val = cst ? C(0,0,0) : C(0,qx,e);
+            }
+            QQ[qx] = W(qx) * coeff_val * detJ(qx,e);
+         }
+         for (int dx = 0; dx < d; ++dx)
+         {
+            double u = 0;
+            for (int qx = 0; qx < q; ++qx) { u += QQ[qx] * B(qx,dx); }
+            Y(dx,c,e) += u;
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0> static
+void BLFEvalAssemble3D(const int vdim, const int nbe, const int d, const int q,
+                       const bool normals, const int *markers, const double *b,
+                       const double *detj, const double *n, const double *weights,
+                       const Vector &coeff, double *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, nbe);
+   const auto B = Reshape(b, q, d);
+   const auto detJ = Reshape(detj, q, q, nbe);
+   const auto N = Reshape(n, q, q, 3, nbe);
+   const auto W = Reshape(weights, q, q);
+   const int cvdim = normals ? 3 : 1;
+   const bool cst = coeff.Size() == cvdim;
+   const auto C = cst ? Reshape(F,cvdim,1,1,1) : Reshape(F,cvdim,q,q,nbe);
+   auto Y = Reshape(y, d, d, vdim, nbe);
+
+   MFEM_FORALL_2D(e, nbe, q, q, 1,
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : MAX_D1D;
+
+      MFEM_SHARED double sBt[Q*D];
+      MFEM_SHARED double sQQ[Q*Q];
+      MFEM_SHARED double sQD[Q*D];
+
+      const DeviceMatrix Bt(sBt, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, B, sBt);
+
+      const DeviceMatrix QQ(sQQ, q, q);
+      const DeviceMatrix QD(sQD, q, d);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               double coeff_val = 0.0;
+               if (normals)
+               {
+                  for (int cd = 0; cd < 3; ++cd)
+                  {
+                     double cval = cst ? C(cd,0,0,0) : C(cd,x,y,e);
+                     coeff_val += cval * N(x,y,cd,e);
+                  }
+               }
+               else
+               {
+                  coeff_val = cst ? C(0,0,0,0) : C(0,x,y,e);
+               }
+               QQ(y,x) = W(x,y) * coeff_val * detJ(x,y,e);
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,d)
+            {
+               double u = 0.0;
+               for (int qx = 0; qx < q; ++qx) { u += QQ(qy,qx) * Bt(dx,qx); }
+               QD(qy,dx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,d)
+         {
+            MFEM_FOREACH_THREAD(dx,x,d)
+            {
+               double u = 0.0;
+               for (int qy = 0; qy < q; ++qy) { u += QD(qy,dx) * Bt(dy,qy); }
+               Y(dx,dy,c,e) += u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+static void BLFEvalAssemble(const FiniteElementSpace &fes,
+                            const IntegrationRule &ir,
+                            const Array<int> &markers,
+                            const Vector &coeff,
+                            const bool normals,
+                            Vector &y)
+{
+   Mesh &mesh = *fes.GetMesh();
+   const int dim = mesh.Dimension();
+   const FiniteElement &el = *fes.GetBE(0);
+   const MemoryType mt = Device::GetDeviceMemoryType();
+   const DofToQuad &maps = el.GetDofToQuad(ir, DofToQuad::TENSOR);
+   const int d = maps.ndof, q = maps.nqpt;
+   int flags = FaceGeometricFactors::DETERMINANTS;
+   if (normals) { flags |= FaceGeometricFactors::NORMALS; }
+   const FaceGeometricFactors *geom = mesh.GetFaceGeometricFactors(
+                                         ir, flags, FaceType::Boundary, mt);
+   auto ker = (dim == 2) ? BLFEvalAssemble2D<> : BLFEvalAssemble3D<>;
+
+   if (dim==2)
+   {
+      if (d==1 && q==1) { ker=BLFEvalAssemble2D<1,1>; }
+      if (d==2 && q==2) { ker=BLFEvalAssemble2D<2,2>; }
+      if (d==3 && q==3) { ker=BLFEvalAssemble2D<3,3>; }
+      if (d==4 && q==4) { ker=BLFEvalAssemble2D<4,4>; }
+      if (d==5 && q==5) { ker=BLFEvalAssemble2D<5,5>; }
+      if (d==2 && q==3) { ker=BLFEvalAssemble2D<2,3>; }
+      if (d==3 && q==4) { ker=BLFEvalAssemble2D<3,4>; }
+      if (d==4 && q==5) { ker=BLFEvalAssemble2D<4,5>; }
+      if (d==5 && q==6) { ker=BLFEvalAssemble2D<5,6>; }
+   }
+
+   if (dim==3)
+   {
+      if (d==1 && q==1) { ker=BLFEvalAssemble3D<1,1>; }
+      if (d==2 && q==2) { ker=BLFEvalAssemble3D<2,2>; }
+      if (d==3 && q==3) { ker=BLFEvalAssemble3D<3,3>; }
+      if (d==4 && q==4) { ker=BLFEvalAssemble3D<4,4>; }
+      if (d==5 && q==5) { ker=BLFEvalAssemble3D<5,5>; }
+      if (d==2 && q==3) { ker=BLFEvalAssemble3D<2,3>; }
+      if (d==3 && q==4) { ker=BLFEvalAssemble3D<3,4>; }
+      if (d==4 && q==5) { ker=BLFEvalAssemble3D<4,5>; }
+      if (d==5 && q==6) { ker=BLFEvalAssemble3D<5,6>; }
+   }
+
+   MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q);
+
+   const int vdim = fes.GetVDim();
+   const int nbe = fes.GetMesh()->GetNFbyType(FaceType::Boundary);
+   const int *M = markers.Read();
+   const double *B = maps.B.Read();
+   const double *detJ = geom->detJ.Read();
+   const double *n = geom->normal.Read();
+   const double *W = ir.GetWeights().Read();
+   double *Y = y.ReadWrite();
+   ker(vdim, nbe, d, q, normals, M, B, detJ, n, W, coeff, Y);
+}
+
+void BoundaryLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
+                                          const Array<int> &markers,
+                                          Vector &b)
+{
+   const FiniteElement &fe = *fes.GetBE(0);
+   const int qorder = oa * fe.GetOrder() + ob;
+   const Geometry::Type gtype = fe.GetGeomType();
+   const IntegrationRule &ir = IntRule ? *IntRule : IntRules.Get(gtype, qorder);
+   Mesh &mesh = *fes.GetMesh();
+
+   FaceQuadratureSpace qs(mesh, ir, FaceType::Boundary);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   BLFEvalAssemble(fes, ir, markers, coeff, false, b);
+}
+
+void BoundaryNormalLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
+                                                const Array<int> &markers,
+                                                Vector &b)
+{
+   const FiniteElement &fe = *fes.GetBE(0);
+   const int qorder = oa * fe.GetOrder() + ob;
+   const Geometry::Type gtype = fe.GetGeomType();
+   const IntegrationRule &ir = IntRule ? *IntRule : IntRules.Get(gtype, qorder);
+   Mesh &mesh = *fes.GetMesh();
+
+   FaceQuadratureSpace qs(mesh, ir, FaceType::Boundary);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   BLFEvalAssemble(fes, ir, markers, coeff, true, b);
+}
+
+} // namespace mfem

fem/lininteg_domain.cpp

@@ -19,13 +19,13 @@ namespace mfem
 template<int T_D1D = 0, int T_Q1D = 0> static
 void DLFEvalAssemble2D(const int vdim, const int ne, const int d, const int q,
                        const int map_type, const int *markers, const double *b,
-                       const double *j, const double *weights,
+                       const double *detj, const double *weights,
                        const Vector &coeff, double *y)
 {
    const auto F = coeff.Read();
    const auto M = Reshape(markers, ne);
    const auto B = Reshape(b, q, d);
-   const auto J = Reshape(j, q, q, 2,2, ne);
+   const auto DETJ = Reshape(detj, q, q, ne);
    const auto W = Reshape(weights, q, q);
    const bool cst = coeff.Size() == vdim;
    const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
@@ -55,19 +55,7 @@ void DLFEvalAssemble2D(const int vdim, const int ne, const int d, const int q,
          {
             MFEM_FOREACH_THREAD(y,y,q)
             {
-               double detJ;
-               if (map_type == FiniteElement::VALUE)
-               {
-                  const double J11 = J(x,y,0,0,e);
-                  const double J21 = J(x,y,1,0,e);
-                  const double J12 = J(x,y,0,1,e);
-                  const double J22 = J(x,y,1,1,e);
-                  detJ = J11 * J22 - J21 * J12;
-               }
-               else
-               {
-                  detJ = 1.0;
-               }
+               const double detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,e) : 1.0;
                const double coeff_val = cst ? cst_val : C(c,x,y,e);
                QQ(y,x) = W(x,y) * coeff_val * detJ;
             }
@@ -100,13 +88,13 @@ void DLFEvalAssemble2D(const int vdim, const int ne, const int d, const int q,
 template<int T_D1D = 0, int T_Q1D = 0> static
 void DLFEvalAssemble3D(const int vdim, const int ne, const int d, const int q,
                        const int map_type, const int *markers, const double *b,
-                       const double *j, const double *weights,
+                       const double *detj, const double *weights,
                        const Vector &coeff, double *y)
 {
    const auto F = coeff.Read();
    const auto M = Reshape(markers, ne);
    const auto B = Reshape(b, q,d);
-   const auto J = Reshape(j, q,q,q, 3,3, ne);
+   const auto DETJ = Reshape(detj, q, q, q, ne);
    const auto W = Reshape(weights, q,q,q);
    const bool cst_coeff = coeff.Size() == vdim;
    const auto C = cst_coeff ? Reshape(F,vdim,1,1,1,1):Reshape(F,vdim,q,q,q,ne);
@@ -138,26 +126,7 @@ void DLFEvalAssemble3D(const int vdim, const int ne, const int d, const int q,
             {
                for (int z = 0; z < q; ++z)
                {
-                  double detJ;
-                  if (map_type == FiniteElement::VALUE)
-                  {
-                     const double J11 = J(x,y,z,0,0,e);
-                     const double J21 = J(x,y,z,1,0,e);
-                     const double J31 = J(x,y,z,2,0,e);
-                     const double J12 = J(x,y,z,0,1,e);
-                     const double J22 = J(x,y,z,1,1,e);
-                     const double J32 = J(x,y,z,2,1,e);
-                     const double J13 = J(x,y,z,0,2,e);
-                     const double J23 = J(x,y,z,1,2,e);
-                     const double J33 = J(x,y,z,2,2,e);
-                     detJ = J11 * (J22 * J33 - J32 * J23) -
-                     /* */  J21 * (J12 * J33 - J32 * J13) +
-                     /* */  J31 * (J12 * J23 - J22 * J13);
-                  }
-                  else
-                  {
-                     detJ = 1.0;
-                  }
+                  const double detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,z,e) : 1.0;
                   const double coeff_val = cst_coeff ? cst_val : C(c,x,y,z,e);
                   QQQ(z,y,x) = W(x,y,z) * coeff_val * detJ;
                }
@@ -222,7 +191,7 @@ static void DLFEvalAssemble(const FiniteElementSpace &fes,
    const MemoryType mt = Device::GetDeviceMemoryType();
    const DofToQuad &maps = el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    const int d = maps.ndof, q = maps.nqpt;
-   constexpr int flags = GeometricFactors::JACOBIANS;
+   constexpr int flags = GeometricFactors::DETERMINANTS;
    const GeometricFactors *geom = mesh->GetGeometricFactors(*ir, flags, mt);
    const int map_type = fes.GetFE(0)->GetMapType();
    decltype(&DLFEvalAssemble2D<>) ker =
@@ -260,10 +229,10 @@ static void DLFEvalAssemble(const FiniteElementSpace &fes,
    const int ne = fes.GetMesh()->GetNE();
    const int *M = markers.Read();
    const double *B = maps.B.Read();
-   const double *J = geom->J.Read();
+   const double *detJ = geom->detJ.Read();
    const double *W = ir->GetWeights().Read();
    double *Y = y.ReadWrite();
-   ker(vdim, ne, d, q, map_type, M, B, J, W, coeff, Y);
+   ker(vdim, ne, d, q, map_type, M, B, detJ, W, coeff, Y);
 }
 
 void DomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
@@ -274,42 +243,9 @@ void DomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
    const int qorder = oa * fe.GetOrder() + ob;
    const Geometry::Type gtype = fe.GetGeomType();
    const IntegrationRule *ir = IntRule ? IntRule : &IntRules.Get(gtype, qorder);
-   const int nq = ir->GetNPoints(), ne = fes.GetMesh()->GetNE();
 
-   Vector coeff;
-   if (ConstantCoefficient *cQ =
-          dynamic_cast<ConstantCoefficient*>(&Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient *qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == fes.GetVDim()*ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& Tr = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            const IntegrationPoint &ip = ir->IntPoint(q);
-            Tr.SetIntPoint(&ip);
-            C(q,e) = Q.Eval(Tr, ip);
-         }
-      }
-   }
+   QuadratureSpace qs(*fes.GetMesh(), *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
    DLFEvalAssemble(fes, ir, markers, coeff, b);
 }
 
@@ -317,48 +253,14 @@ void VectorDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
                                               const Array<int> &markers,
                                               Vector &b)
 {
-   const int vdim = fes.GetVDim();
    const FiniteElement &fe = *fes.GetFE(0);
    const int qorder = 2 * fe.GetOrder();
    const Geometry::Type gtype = fe.GetGeomType();
    const IntegrationRule *ir = IntRule ? IntRule : &IntRules.Get(gtype, qorder);
-   const int nq = ir->GetNPoints(), ne = fes.GetMesh()->GetNE();
 
-   if (VectorConstantCoefficient *vcQ =
-          dynamic_cast<VectorConstantCoefficient*>(&Q))
-   {
-      Qvec = vcQ->GetVec();
-   }
-   else if (VectorQuadratureFunctionCoefficient *vQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = vQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == vdim*ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      Qvec.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else
-   {
-      Vector qv(vdim);
-      Qvec.SetSize(vdim * nq * ne);
-      auto C = Reshape(Qvec.HostWrite(), vdim, nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& Tr = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            const IntegrationPoint &ip = ir->IntPoint(q);
-            Tr.SetIntPoint(&ip);
-            Q.Eval(qv, Tr, ip);
-            for (int c=0; c<vdim; ++c) { C(c,q,e) = qv[c]; }
-         }
-      }
-   }
-   DLFEvalAssemble(fes, ir, markers, Qvec, b);
+   QuadratureSpace qs(*fes.GetMesh(), *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   DLFEvalAssemble(fes, ir, markers, coeff, b);
 }
 
 } // namespace mfem

fem/lininteg_domain_grad.cpp

@@ -324,108 +324,24 @@ void DomainLFGradIntegrator::AssembleDevice(const FiniteElementSpace &fes,
    const int qorder = 2 * fe.GetOrder();
    const Geometry::Type gtype = fe.GetGeomType();
    const IntegrationRule *ir = IntRule ? IntRule : &IntRules.Get(gtype, qorder);
-   const int nq = ir->GetNPoints(), ne = fes.GetMesh()->GetNE();
 
-   if (VectorConstantCoefficient *vcQ =
-          dynamic_cast<VectorConstantCoefficient*>(&Q))
-   {
-      Qvec = vcQ->GetVec();
-   }
-   else if (VectorQuadratureFunctionCoefficient *vqfQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = vqfQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      Qvec.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else
-   {
-      const int qvdim = Q.GetVDim();
-      Vector qvec(qvdim);
-      Qvec.SetSize(qvdim * nq * ne);
-      auto C = Reshape(Qvec.HostWrite(), qvdim, nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& Tr = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            const IntegrationPoint &ip = ir->IntPoint(q);
-            Tr.SetIntPoint(&ip);
-            Q.Eval(qvec, Tr, ip);
-            for (int c=0; c < qvdim; ++c)
-            {
-               C(c,q,e) = qvec[c];
-            }
-         }
-      }
-   }
-   DLFGradAssemble(fes, ir, markers, Qvec, b);
+   QuadratureSpace qs(*fes.GetMesh(), *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   DLFGradAssemble(fes, ir, markers, coeff, b);
 }
 
 void VectorDomainLFGradIntegrator::AssembleDevice(const FiniteElementSpace &fes,
                                                   const Array<int> &markers,
                                                   Vector &b)
 {
-   const int vdim = fes.GetVDim();
    const FiniteElement &fe = *fes.GetFE(0);
    const int qorder = 2 * fe.GetOrder();
    const Geometry::Type gtype = fe.GetGeomType();
    const IntegrationRule *ir = IntRule ? IntRule : &IntRules.Get(gtype, qorder);
-   const int nq = ir->GetNPoints(), ne = fes.GetMesh()->GetNE(),
-             ns = fes.GetMesh()->SpaceDimension();
 
-   if (VectorConstantCoefficient *vcQ =
-          dynamic_cast<VectorConstantCoefficient*>(&Q))
-   {
-      Qvec = vcQ->GetVec();
-   }
-   else if (QuadratureFunctionCoefficient *qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qfun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qfun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different.\n");
-      qfun.Read();
-      Qvec.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
-   }
-   else if (VectorQuadratureFunctionCoefficient* vqfQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
-   {
-      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == vdim * ns * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      Qvec.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      Vector qvec(vdim);
-      Qvec.SetSize(vdim * nq * ne);
-      auto C = Reshape(Qvec.HostWrite(), vdim, nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation &Tr = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            const IntegrationPoint &ip = ir->IntPoint(q);
-            Tr.SetIntPoint(&ip);
-            Q.Eval(qvec, Tr, ip);
-            for (int c = 0; c<vdim; ++c) { C(c,q,e) = qvec[c]; }
-         }
-      }
-   }
-   DLFGradAssemble(fes, ir, markers, Qvec, b);
+   QuadratureSpace qs(*fes.GetMesh(), *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   DLFGradAssemble(fes, ir, markers, coeff, b);
 }
 
 } // namespace mfem

fem/lor/lor.cpp

@@ -468,7 +468,8 @@ void LORDiscretization::FormLORSpace()
 
    fec = fes_ho.FEColl()->Clone(GetLOROrder());
    const int vdim = fes_ho.GetVDim();
-   fes = new FiniteElementSpace(mesh, fec, vdim);
+   const Ordering::Type ordering = fes_ho.GetOrdering();
+   fes = new FiniteElementSpace(mesh, fec, vdim, ordering);
    SetupProlongationAndRestriction();
 }
 
@@ -513,8 +514,8 @@ void ParLORDiscretization::FormLORSpace()
 
    fec = pfes_ho.FEColl()->Clone(GetLOROrder());
    const int vdim = fes_ho.GetVDim();
-   ParFiniteElementSpace *pfes = new ParFiniteElementSpace(pmesh, fec, vdim);
-   fes = pfes;
+   const Ordering::Type ordering = fes_ho.GetOrdering();
+   fes = new ParFiniteElementSpace(pmesh, fec, vdim, ordering);
    SetupProlongationAndRestriction();
 }
 

fem/lor/lor.hpp

@@ -95,7 +95,7 @@ protected:
    /// Returns the order of the LOR space. 1 for H1 or ND, 0 for L2 or RT.
    int GetLOROrder() const;
 
-   /// Construct the LOR space (overriden for serial and parallel versions).
+   /// Construct the LOR space (overridden for serial and parallel versions).
    virtual void FormLORSpace() = 0;
 
    /// Construct the LORBase object for the given FE space and refinement type.

fem/lor/lor_batched.cpp

@@ -91,9 +91,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
    Vector nodal_evec(nodal_restriction->Height());
    nodal_restriction->Mult(*nodal_gf, nodal_evec);
 
-   IntegrationRules irs(0, Quadrature1D::GaussLobatto);
-   Geometry::Type geom = mesh_ho.GetElementGeometry(0);
-   const IntegrationRule &ir = irs.Get(geom, 2*nd1d - 3);
+   IntegrationRule ir = GetCollocatedIntRule(fes_ho);
 
    // Map from nodal E-vector to Q-vector at the LOR vertex points
    X_vert.SetSize(dim*ndof_per_el*nel_ho);
@@ -493,4 +491,12 @@ BatchedLORAssembly::BatchedLORAssembly(FiniteElementSpace &fes_ho_)
    FormLORVertexCoordinates(fes_ho, X_vert);
 }
 
+IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes)
+{
+   IntegrationRules irs(0, Quadrature1D::GaussLobatto);
+   const Geometry::Type geom = fes.GetMesh()->GetElementGeometry(0);
+   const int nd1d = fes.GetMaxElementOrder() + 1;
+   return irs.Get(geom, 2*nd1d - 3);
+}
+
 } // namespace mfem

fem/lor/lor_batched.hpp

@@ -13,6 +13,7 @@
 #define MFEM_LOR_BATCHED
 
 #include "lor.hpp"
+#include "../qspace.hpp"
 
 namespace mfem
 {
@@ -143,6 +144,25 @@ static T *GetIntegrator(BilinearForm &a)
    return nullptr;
 }
 
+IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes);
+
+template <typename INTEGRATOR>
+void ProjectLORCoefficient(BilinearForm &a, CoefficientVector &coeff_vector)
+{
+   INTEGRATOR *i = GetIntegrator<INTEGRATOR>(a);
+   if (i)
+   {
+      // const_cast since Coefficient::Eval is not const...
+      auto *coeff = const_cast<Coefficient*>(i->GetCoefficient());
+      if (coeff) { coeff_vector.Project(*coeff); }
+      else { coeff_vector.SetConstant(1.0); }
+   }
+   else
+   {
+      coeff_vector.SetConstant(0.0);
+   }
+}
+
 /// Abstract base class for the batched LOR assembly kernels.
 class BatchedLORKernel
 {
@@ -151,12 +171,18 @@ protected:
    Vector &X_vert; ///< Mesh coordinate vector.
    Vector &sparse_ij; ///< Local element sparsity matrix data.
    Array<int> &sparse_mapping; ///< Local element sparsity pattern.
+   IntegrationRule ir; ///< Collocated integration rule.
+   QuadratureSpace qs; ///< Quadrature space for coefficients.
+   CoefficientVector c1; ///< Coefficient of first integrator.
+   CoefficientVector c2; ///< Coefficient of second integrator.
    BatchedLORKernel(FiniteElementSpace &fes_ho_,
                     Vector &X_vert_,
                     Vector &sparse_ij_,
                     Array<int> &sparse_mapping_)
       : fes_ho(fes_ho_), X_vert(X_vert_), sparse_ij(sparse_ij_),
-        sparse_mapping(sparse_mapping_)
+        sparse_mapping(sparse_mapping_), ir(GetCollocatedIntRule(fes_ho)),
+        qs(*fes_ho.GetMesh(), ir), c1(qs, CoefficientStorage::COMPRESSED),
+        c2(qs, CoefficientStorage::COMPRESSED)
    { }
 };
 

fem/lor/lor_h1.cpp

@@ -30,8 +30,14 @@ void BatchedLOR_H1::Assemble2D()
    static constexpr int nnz_per_row = 9;
    static constexpr int sz_local_mat = nv*nv;
 
-   const double DQ = diffusion_coeff;
-   const double MQ = mass_coeff;
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1)
+                   : Reshape(c1.Read(), nd1d, nd1d, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1)
+                   : Reshape(c2.Read(), nd1d, nd1d, nel_ho);
 
    sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
    auto V = Reshape(sparse_ij.Write(), nnz_per_row, nd1d, nd1d, nel_ho);
@@ -97,6 +103,8 @@ void BatchedLOR_H1::Assemble2D()
             {
                for (int iqy=0; iqy<2; ++iqy)
                {
+                  const double mq = const_mq ? MQ(0,0,0) : MQ(kx+iqx, ky+iqy, iel_ho);
+                  const double dq = const_dq ? DQ(0,0,0) : DQ(kx+iqx, ky+iqy, iel_ho);
                   for (int jy=0; jy<2; ++jy)
                   {
                      const double bjy = (jy == iqy) ? 1.0 : 0.0;
@@ -133,9 +141,9 @@ void BatchedLOR_H1::Assemble2D()
                               val += dix*djx*Q(0,iqy,iqx);
                               val += (dix*djy + diy*djx)*Q(1,iqy,iqx);
                               val += diy*djy*Q(2,iqy,iqx);
-                              val *= DQ;
+                              val *= dq;
 
-                              val += MQ*bix*biy*bjx*bjy*Q(3,iqy,iqx);
+                              val += mq*bix*biy*bjx*bjy*Q(3,iqy,iqx);
 
                               local_mat(ii_loc, jj_loc) += val;
                            }
@@ -201,10 +209,6 @@ template <int ORDER>
 void BatchedLOR_H1::Assemble3D()
 {
    const int nel_ho = fes_ho.GetNE();
-
-   const double DQ = diffusion_coeff;
-   const double MQ = mass_coeff;
-
    static constexpr int nv = 8;
    static constexpr int dim = 3;
    static constexpr int ddm2 = (dim*(dim+1))/2;
@@ -217,6 +221,15 @@ void BatchedLOR_H1::Assemble3D()
    static constexpr int sz_mass_B = sz_mass_A*2;
    static constexpr int sz_local_mat = nv*nv;
 
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1, 1)
+                   : Reshape(c1.Read(), nd1d, nd1d, nd1d, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1, 1)
+                   : Reshape(c2.Read(), nd1d, nd1d, nd1d, nel_ho);
+
    sparse_ij.SetSize(nel_ho*ndof_per_el*nnz_per_row);
    auto V = Reshape(sparse_ij.Write(), nnz_per_row, nd1d, nd1d, nd1d, nel_ho);
 
@@ -288,7 +301,6 @@ void BatchedLOR_H1::Assemble3D()
                      //MFEM_UNROLL(2)
                      for (int iqx=0; iqx<2; ++iqx)
                      {
-
                         const double x = iqx;
                         const double y = iqy;
                         const double z = iqz;
@@ -335,6 +347,9 @@ void BatchedLOR_H1::Assemble3D()
                            //MFEM_UNROLL(2)
                            for (int iqz=0; iqz<2; ++iqz)
                            {
+                              const double mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
+                              const double dq = const_dq ? DQ(0,0,0,0) : DQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
+
                               const double biz = (iz == iqz) ? 1.0 : 0.0;
                               const double giz = (iz == 0) ? -1.0 : 1.0;
 
@@ -351,18 +366,18 @@ void BatchedLOR_H1::Assemble3D()
                               const double J23 = J32;
                               const double J33 = Q(5,iqz,iqy,iqx);
 
-                              grad_A(0,0,iqy,iz,jz,iqx) += J11*biz*bjz;
-                              grad_A(1,0,iqy,iz,jz,iqx) += J21*biz*bjz;
-                              grad_A(2,0,iqy,iz,jz,iqx) += J31*giz*bjz;
-                              grad_A(0,1,iqy,iz,jz,iqx) += J12*biz*bjz;
-                              grad_A(1,1,iqy,iz,jz,iqx) += J22*biz*bjz;
-                              grad_A(2,1,iqy,iz,jz,iqx) += J32*giz*bjz;
-                              grad_A(0,2,iqy,iz,jz,iqx) += J13*biz*gjz;
-                              grad_A(1,2,iqy,iz,jz,iqx) += J23*biz*gjz;
-                              grad_A(2,2,iqy,iz,jz,iqx) += J33*giz*gjz;
+                              grad_A(0,0,iqy,iz,jz,iqx) += dq*J11*biz*bjz;
+                              grad_A(1,0,iqy,iz,jz,iqx) += dq*J21*biz*bjz;
+                              grad_A(2,0,iqy,iz,jz,iqx) += dq*J31*giz*bjz;
+                              grad_A(0,1,iqy,iz,jz,iqx) += dq*J12*biz*bjz;
+                              grad_A(1,1,iqy,iz,jz,iqx) += dq*J22*biz*bjz;
+                              grad_A(2,1,iqy,iz,jz,iqx) += dq*J32*giz*bjz;
+                              grad_A(0,2,iqy,iz,jz,iqx) += dq*J13*biz*gjz;
+                              grad_A(1,2,iqy,iz,jz,iqx) += dq*J23*biz*gjz;
+                              grad_A(2,2,iqy,iz,jz,iqx) += dq*J33*giz*gjz;
 
                               double wdetJ = Q(6,iqz,iqy,iqx);
-                              mass_A(iqy,iz,jz,iqx) += wdetJ*biz*bjz;
+                              mass_A(iqy,iz,jz,iqx) += mq*wdetJ*biz*bjz;
                            }
                            //MFEM_UNROLL(2)
                            for (int jy=0; jy<2; ++jy)
@@ -426,9 +441,7 @@ void BatchedLOR_H1::Assemble3D()
                                     val += bix*bjx*grad_B(2,2,iy,jy,iz,jz,iqx);
                                     val += bix*bjx*grad_B(1,2,iy,jy,iz,jz,iqx);
 
-                                    val *= DQ;
-
-                                    val += MQ*bix*bjx*mass_B(iy,jy,iz,jz,iqx);
+                                    val += bix*bjx*mass_B(iy,jy,iz,jz,iqx);
 
                                     local_mat(ii_loc, jj_loc) += val;
                                  }
@@ -531,29 +544,8 @@ BatchedLOR_H1::BatchedLOR_H1(BilinearForm &a,
                              Array<int> &sparse_mapping_)
    : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
 {
-   MassIntegrator *mass = GetIntegrator<MassIntegrator>(a);
-   DiffusionIntegrator *diffusion = GetIntegrator<DiffusionIntegrator>(a);
-
-   if (mass != nullptr)
-   {
-      auto *coeff = dynamic_cast<const ConstantCoefficient*>(mass->GetCoefficient());
-      mass_coeff = coeff ? coeff->constant : 1.0;
-   }
-   else
-   {
-      mass_coeff = 0.0;
-   }
-
-   if (diffusion != nullptr)
-   {
-      auto *coeff = dynamic_cast<const ConstantCoefficient*>
-                    (diffusion->GetCoefficient());
-      diffusion_coeff = coeff ? coeff->constant : 1.0;
-   }
-   else
-   {
-      diffusion_coeff = 0.0;
-   }
+   ProjectLORCoefficient<MassIntegrator>(a, c1);
+   ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
 }
 
 } // namespace mfem

fem/lor/lor_h1.hpp

@@ -21,9 +21,6 @@ namespace mfem
 // classes BatchedLORAssembly and BatchedLORKernel .
 class BatchedLOR_H1 : BatchedLORKernel
 {
-protected:
-   // TODO: for now only supporting constant coefficients
-   double mass_coeff, diffusion_coeff;
 public:
    template <int ORDER> void Assemble2D();
    template <int ORDER> void Assemble3D();

fem/lor/lor_nd.cpp

@@ -33,8 +33,14 @@ void BatchedLOR_ND::Assemble2D()
    static constexpr int nnz_per_row = 7;
    static constexpr int sz_local_mat = ne*ne;
 
-   const double DQ = curl_curl_coeff;
-   const double MQ = mass_coeff;
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1)
+                   : Reshape(c1.Read(), op1, op1, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1)
+                   : Reshape(c2.Read(), op1, op1, nel_ho);
 
    sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
    auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*op1, dim, nel_ho);
@@ -106,6 +112,8 @@ void BatchedLOR_ND::Assemble2D()
             {
                for (int iqy=0; iqy<2; ++iqy)
                {
+                  const double mq = const_mq ? MQ(0,0,0) : MQ(kx+iqx, ky+iqy, iel_ho);
+                  const double dq = const_dq ? DQ(0,0,0) : DQ(kx+iqx, ky+iqy, iel_ho);
                   // Loop over x,y components. c=0 => x, c=1 => y
                   for (int cj=0; cj<dim; ++cj)
                   {
@@ -136,8 +144,8 @@ void BatchedLOR_ND::Assemble2D()
                               val += byi*bxj*Q(1,iqy,iqx);
                               val += bxi*byj*Q(1,iqy,iqx);
                               val += byi*byj*Q(2,iqy,iqx);
-                              val *= MQ;
-                              val += DQ*curl_i*curl_j*Q(3,iqy,iqx);
+                              val *= mq;
+                              val += dq*curl_i*curl_j*Q(3,iqy,iqx);
 
                               local_mat(ii_loc, jj_loc) += val;
                            }
@@ -224,8 +232,14 @@ void BatchedLOR_ND::Assemble3D()
    static constexpr int nnz_per_row = 33;
    static constexpr int sz_local_mat = ne*ne;
 
-   const double DQ = curl_curl_coeff;
-   const double MQ = mass_coeff;
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1, 1)
+                   : Reshape(c1.Read(), op1, op1, op1, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1, 1)
+                   : Reshape(c2.Read(), op1, op1, op1, nel_ho);
 
    sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
    auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*op1*op1, dim, nel_ho);
@@ -318,6 +332,8 @@ void BatchedLOR_ND::Assemble3D()
                   {
                      for (int iqx=0; iqx<2; ++iqx)
                      {
+                        const double mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
+                        const double dq = const_dq ? DQ(0,0,0,0) : DQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
                         // Loop over x,y,z components. 0 => x, 1 => y, 2 => z
                         for (int cj=0; cj<dim; ++cj)
                         {
@@ -391,7 +407,7 @@ void BatchedLOR_ND::Assemble3D()
                                     basis_basis += Q(4,iqz,iqy,iqx)*(basis_i[1]*basis_j[2] + basis_i[2]*basis_j[1]);
                                     basis_basis += Q(5,iqz,iqy,iqx)*basis_i[2]*basis_j[2];
 
-                                    const double val = DQ*curl_curl + MQ*basis_basis;
+                                    const double val = dq*curl_curl + mq*basis_basis;
 
                                     local_mat(ii_loc, jj_loc) += val;
                                  }
@@ -572,28 +588,8 @@ BatchedLOR_ND::BatchedLOR_ND(BilinearForm &a,
                              Array<int> &sparse_mapping_)
    : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
 {
-   VectorFEMassIntegrator *mass = GetIntegrator<VectorFEMassIntegrator>(a);
-   if (mass != nullptr)
-   {
-      auto *coeff = dynamic_cast<const ConstantCoefficient*>(mass->GetCoefficient());
-      mass_coeff = coeff ? coeff->constant : 1.0;
-   }
-   else
-   {
-      mass_coeff = 0.0;
-   }
-
-   CurlCurlIntegrator *diffusion = GetIntegrator<CurlCurlIntegrator>(a);
-   if (diffusion != nullptr)
-   {
-      auto *coeff = dynamic_cast<const ConstantCoefficient*>
-                    (diffusion->GetCoefficient());
-      curl_curl_coeff = coeff ? coeff->constant : 1.0;
-   }
-   else
-   {
-      curl_curl_coeff = 0.0;
-   }
+   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+   ProjectLORCoefficient<CurlCurlIntegrator>(a, c2);
 }
 
 } // namespace mfem

fem/lor/lor_nd.hpp

@@ -21,8 +21,6 @@ namespace mfem
 // classes BatchedLORAssembly and BatchedLORKernel .
 class BatchedLOR_ND : BatchedLORKernel
 {
-protected:
-   double mass_coeff, curl_curl_coeff;
 public:
    template <int ORDER> void Assemble2D();
    template <int ORDER> void Assemble3D();

fem/lor/lor_rt.cpp

@@ -33,8 +33,14 @@ void BatchedLOR_RT::Assemble2D()
    static constexpr int nnz_per_row = 7;
    static constexpr int sz_local_mat = ne*ne;
 
-   const double DQ = div_div_coeff;
-   const double MQ = mass_coeff;
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1)
+                   : Reshape(c1.Read(), op1, op1, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1)
+                   : Reshape(c2.Read(), op1, op1, nel_ho);
 
    sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
    auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*op1, dim, nel_ho);
@@ -102,6 +108,8 @@ void BatchedLOR_RT::Assemble2D()
             {
                for (int iqy=0; iqy<2; ++iqy)
                {
+                  const double mq = const_mq ? MQ(0,0,0) : MQ(kx+iqx, ky+iqy, iel_ho);
+                  const double dq = const_dq ? DQ(0,0,0) : DQ(kx+iqx, ky+iqy, iel_ho);
                   // Loop over x,y components. c=0 => x, c=1 => y
                   for (int cj=0; cj<dim; ++cj)
                   {
@@ -132,8 +140,8 @@ void BatchedLOR_RT::Assemble2D()
                               val += byi*bxj*Q(1,iqy,iqx);
                               val += bxi*byj*Q(1,iqy,iqx);
                               val += byi*byj*Q(2,iqy,iqx);
-                              val *= MQ;
-                              val += DQ*div_j*div_i*Q(3,iqy,iqx);
+                              val *= mq;
+                              val += dq*div_j*div_i*Q(3,iqy,iqx);
 
                               local_mat(ii_loc, jj_loc) += val;
                            }
@@ -241,8 +249,14 @@ void BatchedLOR_RT::Assemble3D()
    static constexpr int nnz_per_row = 11;
    static constexpr int sz_local_mat = nf*nf;
 
-   const double DQ = div_div_coeff;
-   const double MQ = mass_coeff;
+   const bool const_mq = c1.Size() == 1;
+   const auto MQ = const_mq
+                   ? Reshape(c1.Read(), 1, 1, 1, 1)
+                   : Reshape(c1.Read(), op1, op1, op1, nel_ho);
+   const bool const_dq = c2.Size() == 1;
+   const auto DQ = const_dq
+                   ? Reshape(c2.Read(), 1, 1, 1, 1)
+                   : Reshape(c2.Read(), op1, op1, op1, nel_ho);
 
    sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
    auto V = Reshape(sparse_ij.Write(), nnz_per_row, o*o*op1, dim, nel_ho);
@@ -323,6 +337,8 @@ void BatchedLOR_RT::Assemble3D()
                   {
                      for (int iqx=0; iqx<2; ++iqx)
                      {
+                        const double mq = const_mq ? MQ(0,0,0,0) : MQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
+                        const double dq = const_dq ? DQ(0,0,0,0) : DQ(kx+iqx, ky+iqy, kz+iqz, iel_ho);
                         // Loop over x,y,z components. 0 => x, 1 => y, 2 => z
                         for (int cj=0; cj<dim; ++cj)
                         {
@@ -376,7 +392,7 @@ void BatchedLOR_RT::Assemble3D()
                                     basis_basis += Q(4,iqz,iqy,iqx)*(basis_i[1]*basis_j[2] + basis_i[2]*basis_j[1]);
                                     basis_basis += Q(5,iqz,iqy,iqx)*basis_i[2]*basis_j[2];
 
-                                    const double val = DQ*div_div + MQ*basis_basis;
+                                    const double val = dq*div_div + mq*basis_basis;
                                     // const double val = 1.0;
 
                                     local_mat(ii_loc, jj_loc) += val;
@@ -556,26 +572,8 @@ BatchedLOR_RT::BatchedLOR_RT(BilinearForm &a,
                              Array<int> &sparse_mapping_)
    : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_)
 {
-   if (VectorFEMassIntegrator *mass = GetIntegrator<VectorFEMassIntegrator>(a))
-   {
-      auto *coeff = dynamic_cast<const ConstantCoefficient*>(mass->GetCoefficient());
-      mass_coeff = coeff ? coeff->constant : 1.0;
-   }
-   else
-   {
-      mass_coeff = 0.0;
-   }
-
-   if (DivDivIntegrator *divdiv = GetIntegrator<DivDivIntegrator>(a))
-   {
-      auto *coeff = dynamic_cast<const ConstantCoefficient*>
-                    (divdiv->GetCoefficient());
-      div_div_coeff = coeff ? coeff->constant : 1.0;
-   }
-   else
-   {
-      div_div_coeff = 0.0;
-   }
+   ProjectLORCoefficient<VectorFEMassIntegrator>(a, c1);
+   ProjectLORCoefficient<DivDivIntegrator>(a, c2);
 }
 
 } // namespace mfem

fem/lor/lor_rt.hpp

@@ -21,8 +21,6 @@ namespace mfem
 // classes BatchedLORAssembly and BatchedLORKernel .
 class BatchedLOR_RT : BatchedLORKernel
 {
-protected:
-   double mass_coeff, div_div_coeff;
 public:
    template <int ORDER> void Assemble2D();
    template <int ORDER> void Assemble3D();

fem/nonlinearform.hpp

@@ -157,14 +157,14 @@ public:
    /// Return a (read-only) list of all essential true dofs.
    const Array<int> &GetEssentialTrueDofs() const { return ess_tdof_list; }
 
-   /// Compute the enery corresponding to the state @a x.
+   /// Compute the energy corresponding to the state @a x.
    /** In general, @a x may have non-homogeneous essential boundary values.
 
        The state @a x must be a "GridFunction size" vector, i.e. its size must
        be fes->GetVSize(). */
    double GetGridFunctionEnergy(const Vector &x) const;
 
-   /// Compute the enery corresponding to the state @a x.
+   /// Compute the energy corresponding to the state @a x.
    /** In general, @a x may have non-homogeneous essential boundary values.
 
        The state @a x must be a true-dof vector. */

fem/pfespace.cpp

@@ -129,7 +129,7 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
       ApplyLDofSigns(*elem_dof);
    }
 
-   // Check for shared trianglular faces with interior Nedelec DoFs
+   // Check for shared triangular faces with interior Nedelec DoFs
    CheckNDSTriaDofs();
 }
 

fem/pfespace.hpp

@@ -90,7 +90,7 @@ private:
    /// Flag indicating the existence of shared triangles with interior ND dofs
    bool nd_strias;
 
-   /// Resets nd_strias flag at constuction or after rebalancing
+   /// Resets nd_strias flag at construction or after rebalancing
    void CheckNDSTriaDofs();
 
    ParNURBSExtension *pNURBSext() const

fem/pgridfunc.hpp

@@ -279,19 +279,25 @@ public:
    { return ComputeLpError(1.0, exsol, NULL, NULL, irs); }
 
    virtual double ComputeL2Error(Coefficient *exsol[],
-                                 const IntegrationRule *irs[] = NULL) const
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const
    {
-      return GlobalLpNorm(2.0, GridFunction::ComputeL2Error(exsol, irs),
+      return GlobalLpNorm(2.0, GridFunction::ComputeL2Error(exsol, irs, elems),
                           pfes->GetComm());
    }
 
    virtual double ComputeL2Error(Coefficient &exsol,
-                                 const IntegrationRule *irs[] = NULL) const
-   { return ComputeLpError(2.0, exsol, NULL, irs); }
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const
+   {
+      return GlobalLpNorm(2.0, GridFunction::ComputeL2Error(exsol, irs, elems),
+                          pfes->GetComm());
+   }
+
 
    virtual double ComputeL2Error(VectorCoefficient &exsol,
                                  const IntegrationRule *irs[] = NULL,
-                                 Array<int> *elems = NULL) const
+                                 const Array<int> *elems = NULL) const
    {
       return GlobalLpNorm(2.0, GridFunction::ComputeL2Error(exsol, irs, elems),
                           pfes->GetComm());
@@ -390,10 +396,11 @@ public:
 
    virtual double ComputeLpError(const double p, Coefficient &exsol,
                                  Coefficient *weight = NULL,
-                                 const IntegrationRule *irs[] = NULL) const
+                                 const IntegrationRule *irs[] = NULL,
+                                 const Array<int> *elems = NULL) const
    {
-      return GlobalLpNorm(p, GridFunction::ComputeLpError(
-                             p, exsol, weight, irs), pfes->GetComm());
+      return GlobalLpNorm(p, GridFunction::ComputeLpError(p, exsol, weight, irs,
+                                                          elems), pfes->GetComm());
    }
 
    /** When given a vector weight, compute the pointwise (scalar) error as the

fem/prestriction.cpp

@@ -42,118 +42,108 @@ ParNCH1FaceRestriction::ParNCH1FaceRestriction(const ParFiniteElementSpace &fes,
 
 void ParNCH1FaceRestriction::Mult(const Vector &x, Vector &y) const
 {
-   if (nf==0) { return; }
+   H1FaceRestriction::Mult(x, y);
+   NonconformingInterpolation(y);
+}
+
+void ParNCH1FaceRestriction::NonconformingInterpolation(Vector& y) const
+{
    // Assumes all elements have the same number of dofs
    const int nface_dofs = face_dofs;
    const int vd = vdim;
-   const bool t = byvdim;
-
-   if ( type==FaceType::Boundary )
+   auto d_y = Reshape(y.ReadWrite(), nface_dofs, vd, nf);
+   auto &nc_interp_config = interpolations.GetNCFaceInterpConfig();
+   const int num_nc_faces = nc_interp_config.Size();
+   if ( num_nc_faces == 0 ) { return; }
+   auto interp_config_ptr = nc_interp_config.Read();
+   const int nc_size = interpolations.GetNumInterpolators();
+   auto d_interp = Reshape(interpolations.GetInterpolators().Read(),
+                           nface_dofs, nface_dofs, nc_size);
+   static constexpr int max_nd = 16*16;
+   MFEM_VERIFY(nface_dofs<=max_nd, "Too many degrees of freedom.");
+   MFEM_FORALL_3D(nc_face, num_nc_faces, nface_dofs, 1, 1,
    {
-      auto d_indices = scatter_indices.Read();
-      auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-      auto d_y = Reshape(y.Write(), nface_dofs, vd, nf);
-      MFEM_FORALL(i, nfdofs,
+      MFEM_SHARED double dof_values[max_nd];
+      const NCInterpConfig conf = interp_config_ptr[nc_face];
+      if ( conf.is_non_conforming && conf.master_side == 0 )
       {
-         const int dof = i % nface_dofs;
-         const int face = i / nface_dofs;
-         const int idx = d_indices[i];
-         for (int c = 0; c < vd; ++c)
-         {
-            d_y(dof, c, face) = d_x(t?c:idx, t?idx:c);
-         }
-      });
-   }
-   else // type==FaceType::Interior
-   {
-      auto d_indices = scatter_indices.Read();
-      auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-      auto d_y = Reshape(y.Write(), nface_dofs, vd, nf);
-      auto interp_config_ptr = interpolations.GetFaceInterpConfig().Read();
-      auto interpolators = interpolations.GetInterpolators().Read();
-      const int nc_size = interpolations.GetNumInterpolators();
-      auto d_interp = Reshape(interpolators, nface_dofs, nface_dofs, nc_size);
-      static constexpr int max_nd = 1024;
-      MFEM_VERIFY(nface_dofs<=max_nd, "Too many degrees of freedom.");
-      MFEM_FORALL_3D(face, nf, nface_dofs, 1, 1,
-      {
-         MFEM_SHARED double dof_values[max_nd];
-         const InterpConfig conf = interp_config_ptr[face];
-         const int master_side = conf.master_side;
          const int interp_index = conf.index;
-         const int side = 0;
-         if ( !conf.is_non_conforming || side!=master_side )
+         const int face = conf.face_index;
+         for (int c = 0; c < vd; ++c)
          {
             MFEM_FOREACH_THREAD(dof,x,nface_dofs)
             {
-               const int i = face*nface_dofs + dof;
-               const int idx = d_indices[i];
-               for (int c = 0; c < vd; ++c)
-               {
-                  d_y(dof, c, face) = d_x(t?c:idx, t?idx:c);
-               }
+               dof_values[dof] = d_y(dof, c, face);
             }
-         }
-         else // Interpolation from coarse to fine
-         {
-            for (int c = 0; c < vd; ++c)
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dof_out,x,nface_dofs)
             {
-               // Load the face dofs in shared memory
-               MFEM_FOREACH_THREAD(dof,x,nface_dofs)
+               double res = 0.0;
+               for (int dof_in = 0; dof_in<nface_dofs; dof_in++)
                {
-                  const int i = face*nface_dofs + dof;
-                  const int idx = d_indices[i];
-                  dof_values[dof] = d_x(t?c:idx, t?idx:c);
+                  res += d_interp(dof_out, dof_in, interp_index)*dof_values[dof_in];
                }
-               MFEM_SYNC_THREAD;
-               // Apply the interpolation to the face dofs
-               MFEM_FOREACH_THREAD(dof_out,x,nface_dofs)
-               {
-                  double res = 0.0;
-                  for (int dof_in = 0; dof_in<nface_dofs; dof_in++)
-                  {
-                     res += d_interp(dof_out, dof_in, interp_index)*
-                            dof_values[dof_in];
-                  }
-                  d_y(dof_out, c, face) = res;
-               }
-               MFEM_SYNC_THREAD;
+               d_y(dof_out, c, face) = res;
             }
+            MFEM_SYNC_THREAD;
          }
-      });
-   }
+      }
+   });
 }
 
 void ParNCH1FaceRestriction::AddMultTranspose(const Vector &x, Vector &y) const
 {
    if (nf==0) { return; }
+   NonconformingTransposeInterpolation(x);
+   H1FaceRestriction::AddMultTranspose(x_interp, y);
+}
+
+void ParNCH1FaceRestriction::AddMultTransposeInPlace(Vector &x, Vector &y) const
+{
+   if (nf==0) { return; }
+   NonconformingTransposeInterpolationInPlace(x);
+   H1FaceRestriction::AddMultTranspose(x, y);
+}
+
+void ParNCH1FaceRestriction::NonconformingTransposeInterpolation(
+   const Vector& x) const
+{
    if (x_interp.Size()==0)
    {
       x_interp.SetSize(x.Size());
    }
    x_interp = x;
+   NonconformingTransposeInterpolationInPlace(x_interp);
+}
+
+void ParNCH1FaceRestriction::NonconformingTransposeInterpolationInPlace(
+   Vector& x) const
+{
    // Assumes all elements have the same number of dofs
    const int nface_dofs = face_dofs;
    const int vd = vdim;
-   const bool t = byvdim;
    if ( type==FaceType::Interior )
    {
       // Interpolation from slave to master face dofs
-      auto d_x = Reshape(x_interp.ReadWrite(), nface_dofs, vd, nf);
-      auto interp_config_ptr = interpolations.GetFaceInterpConfig().Read();
-      auto interpolators = interpolations.GetInterpolators().Read();
+      auto d_x = Reshape(x.ReadWrite(), nface_dofs, vd, nf);
+      auto &nc_interp_config = interpolations.GetNCFaceInterpConfig();
+      const int num_nc_faces = nc_interp_config.Size();
+      if ( num_nc_faces == 0 ) { return; }
+      auto interp_config_ptr = nc_interp_config.Read();
       const int nc_size = interpolations.GetNumInterpolators();
-      auto d_interp = Reshape(interpolators, nface_dofs, nface_dofs, nc_size);
+      auto d_interp = Reshape(interpolations.GetInterpolators().Read(),
+                              nface_dofs, nface_dofs, nc_size);
       static constexpr int max_nd = 1024;
       MFEM_VERIFY(nface_dofs<=max_nd, "Too many degrees of freedom.");
-      MFEM_FORALL_3D(face, nf, nface_dofs, 1, 1,
+      MFEM_FORALL_3D(nc_face, num_nc_faces, nface_dofs, 1, 1,
       {
          MFEM_SHARED double dof_values[max_nd];
-         const InterpConfig conf = interp_config_ptr[face];
+         const NCInterpConfig conf = interp_config_ptr[nc_face];
          const int master_side = conf.master_side;
-         const int interp_index = conf.index;
          if ( conf.is_non_conforming && master_side==0 )
          {
+            const int interp_index = conf.index;
+            const int face = conf.face_index;
             // Interpolation from fine to coarse
             for (int c = 0; c < vd; ++c)
             {
@@ -176,27 +166,6 @@ void ParNCH1FaceRestriction::AddMultTranspose(const Vector &x, Vector &y) const
          }
       });
    }
-
-   // Gathering of face dofs into element dofs
-   auto d_offsets = gather_offsets.Read();
-   auto d_indices = gather_indices.Read();
-   auto d_x = Reshape(x_interp.Read(), nface_dofs, vd, nf);
-   auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
-   MFEM_FORALL(i, ndofs,
-   {
-      const int offset = d_offsets[i];
-      const int next_offset = d_offsets[i + 1];
-      for (int c = 0; c < vd; ++c)
-      {
-         double dof_value = 0;
-         for (int j = offset; j < next_offset; ++j)
-         {
-            int idx_j = d_indices[j];
-            dof_value +=  d_x(idx_j % nface_dofs, c, idx_j / nface_dofs);
-         }
-         d_y(t?c:i,t?i:c) += dof_value;
-      }
-   });
 }
 
 void ParNCH1FaceRestriction::ComputeScatterIndicesAndOffsets(
@@ -264,6 +233,7 @@ void ParNCH1FaceRestriction::ComputeScatterIndicesAndOffsets(
 
    // Transform the interpolation matrix map into a contiguous memory structure.
    interpolations.LinearizeInterpolatorMapIntoVector();
+   interpolations.InitializeNCInterpConfig();
 }
 
 void ParNCH1FaceRestriction::ComputeGatherIndices(
@@ -775,110 +745,8 @@ void ParNCL2FaceRestriction::SingleValuedNonconformingMult(
 void ParNCL2FaceRestriction::DoubleValuedNonconformingMult(
    const Vector& x, Vector& y) const
 {
-   MFEM_ASSERT(
-      m == L2FaceValues::DoubleValued,
-      "This method should be called when m == L2FaceValues::DoubleValued.");
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);
-   ParGridFunction x_gf;
-   x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(&pfes),
-                const_cast<Vector&>(x), 0);
-   x_gf.ExchangeFaceNbrData();
-
-   // Assumes all elements have the same number of dofs
-   const int nface_dofs = face_dofs;
-   const int vd = vdim;
-   const bool t = byvdim;
-   const int threshold = ndofs;
-   const int nsdofs = pfes.GetFaceNbrVSize();
-   auto d_indices1 = scatter_indices1.Read();
-   auto d_indices2 = scatter_indices2.Read();
-   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_x_shared = Reshape(x_gf.FaceNbrData().Read(),
-                             t?vd:nsdofs, t?nsdofs:vd);
-   auto d_y = Reshape(y.Write(), nface_dofs, vd, 2, nf);
-   auto interp_config_ptr = interpolations.GetFaceInterpConfig().Read();
-   auto interpolators = interpolations.GetInterpolators().Read();
-   const int nc_size = interpolations.GetNumInterpolators();
-   auto d_interp = Reshape(interpolators, nface_dofs, nface_dofs, nc_size);
-   static constexpr int max_nd = 1024;
-   MFEM_VERIFY(nface_dofs<=max_nd, "Too many degrees of freedom.");
-   MFEM_FORALL_3D(face, nf, nface_dofs, 1, 1,
-   {
-      MFEM_SHARED double dof_values[max_nd];
-      const InterpConfig conf = interp_config_ptr[face];
-      const int master_side = conf.master_side;
-      const int interp_index = conf.index;
-      for (int side = 0; side < 2; side++)
-      {
-         if ( !conf.is_non_conforming || side!=master_side )
-         {
-            // No interpolation
-            MFEM_FOREACH_THREAD(dof,x,nface_dofs)
-            {
-               const int i = face*nface_dofs + dof;
-               const int idx = side==0 ? d_indices1[i] : d_indices2[i];
-               if (idx>-1 && idx<threshold) // local interior face
-               {
-                  for (int c = 0; c < vd; ++c)
-                  {
-                     d_y(dof, c, side, face) = d_x(t?c:idx, t?idx:c);
-                  }
-               }
-               else if (idx>=threshold) // shared interior face
-               {
-                  const int sidx = idx-threshold;
-                  for (int c = 0; c < vd; ++c)
-                  {
-                     d_y(dof, c, side, face) = d_x_shared(t?c:sidx, t?sidx:c);
-                  }
-               }
-               else // true boundary
-               {
-                  for (int c = 0; c < vd; ++c)
-                  {
-                     d_y(dof, c, side, face) = 0.0;
-                  }
-               }
-            }
-         }
-         else // Interpolation from coarse to fine
-         {
-            for (int c = 0; c < vd; ++c)
-            {
-               MFEM_FOREACH_THREAD(dof,x,nface_dofs)
-               {
-                  const int i = face*nface_dofs + dof;
-                  const int idx = side==0 ? d_indices1[i] : d_indices2[i];
-                  if (idx>-1 && idx<threshold) // local interior face
-                  {
-                     dof_values[dof] = d_x(t?c:idx, t?idx:c);
-                  }
-                  else if (idx>=threshold) // shared interior face
-                  {
-                     const int sidx = idx-threshold;
-                     dof_values[dof] = d_x_shared(t?c:sidx, t?sidx:c);
-                  }
-                  else // true boundary
-                  {
-                     dof_values[dof] = 0.0;
-                  }
-               }
-               MFEM_SYNC_THREAD;
-               MFEM_FOREACH_THREAD(dof_out,x,nface_dofs)
-               {
-                  double res = 0.0;
-                  for (int dof_in = 0; dof_in<nface_dofs; dof_in++)
-                  {
-                     res += d_interp(dof_out, dof_in, interp_index)*dof_values[dof_in];
-                  }
-                  d_y(dof_out, c, side, face) = res;
-               }
-               MFEM_SYNC_THREAD;
-            }
-         }
-      }
-   });
+   ParL2FaceRestriction::DoubleValuedConformingMult(x, y);
+   NCL2FaceRestriction::DoubleValuedNonconformingInterpolation(y);
 }
 
 void ParNCL2FaceRestriction::Mult(const Vector& x, Vector& y) const
@@ -935,6 +803,35 @@ void ParNCL2FaceRestriction::AddMultTranspose(const Vector &x, Vector &y) const
    }
 }
 
+void ParNCL2FaceRestriction::AddMultTransposeInPlace(Vector& x, Vector& y) const
+{
+   if (nf==0) { return; }
+   if (type==FaceType::Interior)
+   {
+      if ( m==L2FaceValues::DoubleValued )
+      {
+         DoubleValuedNonconformingTransposeInterpolationInPlace(x);
+         DoubleValuedConformingAddMultTranspose(x, y);
+      }
+      else if ( m==L2FaceValues::SingleValued )
+      {
+         SingleValuedNonconformingTransposeInterpolationInPlace(x);
+         SingleValuedConformingAddMultTranspose(x, y);
+      }
+   }
+   else
+   {
+      if ( m==L2FaceValues::DoubleValued )
+      {
+         DoubleValuedConformingAddMultTranspose(x, y);
+      }
+      else if ( m==L2FaceValues::SingleValued )
+      {
+         SingleValuedConformingAddMultTranspose(x, y);
+      }
+   }
+}
+
 void ParNCL2FaceRestriction::FillI(SparseMatrix &mat,
                                    const bool keep_nbr_block) const
 {
@@ -1042,6 +939,7 @@ void ParNCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 
    // Transform the interpolation matrix map into a contiguous memory structure.
    interpolations.LinearizeInterpolatorMapIntoVector();
+   interpolations.InitializeNCInterpConfig();
 }
 
 void ParNCL2FaceRestriction::ComputeGatherIndices(

fem/prestriction.hpp

@@ -68,6 +68,21 @@ public:
        @param[in,out] y The L-vector degrees of freedom. */
    void AddMultTranspose(const Vector &x, Vector &y) const override;
 
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in,out]  x The face E-Vector degrees of freedom with the given format:
+                         face_dofs x vdim x nf
+                         where nf is the number of interior or boundary faces
+                         requested by @a type in the constructor.
+                         The face_dofs should be ordered according to the given
+                         ElementDofOrdering.
+       @param[in,out] y The L-vector degrees of freedom.
+
+      @note This method is an optimization of AddMultTranspose where the @a x
+      Vector is used and modified to avoid memory allocation and memcpy. */
+   void AddMultTransposeInPlace(Vector &x, Vector &y) const override;
+
 private:
    /** @brief Compute the scatter indices: L-vector to E-vector, the offsets
        for the gathering: E-vector to L-vector, and the interpolators from
@@ -88,6 +103,31 @@ private:
    */
    void ComputeGatherIndices(const ElementDofOrdering ordering,
                              const FaceType type);
+
+public: // For nvcc
+   /** @brief Apply a change of basis from coarse element basis to fine element
+       basis for the coarse face dofs.
+
+       @param[in,out] x The dofs vector that needs coarse dofs to be express in
+                        term of the fine basis.
+   */
+   void NonconformingInterpolation(Vector& x) const;
+
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs.
+
+       @param[in] x The dofs vector that needs coarse dofs to be express in term
+                    of the coarse basis, the result is stored in x_interp.
+   */
+   void NonconformingTransposeInterpolation(const Vector& x) const;
+
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs.
+
+       @param[in] x The dofs vector that needs coarse dofs to be express in term
+                    of the coarse basis, the result is stored in x_interp.
+   */
+   void NonconformingTransposeInterpolationInPlace(Vector& x) const;
 };
 
 /// Operator that extracts Face degrees of freedom in parallel.
@@ -265,6 +305,21 @@ public:
        @param[in,out] y The L-vector degrees of freedom. */
    void AddMultTranspose(const Vector &x, Vector &y) const override;
 
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in,out]  x The face E-Vector degrees of freedom with the given format:
+                         if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                         if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                         where nf is the number of interior or boundary faces
+                         requested by @a type in the constructor.
+                         The face_dofs should be ordered according to the given
+                         ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom.
+
+       @note @a x is used for computation. */
+   void AddMultTransposeInPlace(Vector &x, Vector &y) const override;
+
    /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
        pattern given by this ParNCL2FaceRestriction.
 

fem/qfunction.cpp

@@ -0,0 +1,265 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "qfunction.hpp"
+#include "quadinterpolator.hpp"
+#include "quadinterpolator_face.hpp"
+
+namespace mfem
+{
+
+QuadratureFunction &QuadratureFunction::operator=(double value)
+{
+   Vector::operator=(value);
+   return *this;
+}
+
+QuadratureFunction &QuadratureFunction::operator=(const Vector &v)
+{
+   MFEM_ASSERT(qspace && v.Size() == this->Size(), "");
+   Vector::operator=(v);
+   return *this;
+}
+
+
+QuadratureFunction::QuadratureFunction(Mesh *mesh, std::istream &in)
+{
+   const char *msg = "invalid input stream";
+   std::string ident;
+
+   qspace = new QuadratureSpace(mesh, in);
+   own_qspace = true;
+
+   in >> ident; MFEM_VERIFY(ident == "VDim:", msg);
+   in >> vdim;
+
+   Load(in, vdim*qspace->GetSize());
+}
+
+void QuadratureFunction::SetSpace(QuadratureSpaceBase *qspace_, int vdim_)
+{
+   if (qspace_ != qspace)
+   {
+      if (own_qspace) { delete qspace; }
+      qspace = qspace_;
+      own_qspace = false;
+   }
+   vdim = (vdim_ < 0) ? vdim : vdim_;
+   SetSize(vdim*qspace->GetSize());
+}
+
+void QuadratureFunction::SetSpace(
+   QuadratureSpaceBase *qspace_, double *qf_data, int vdim_)
+{
+   if (qspace_ != qspace)
+   {
+      if (own_qspace) { delete qspace; }
+      qspace = qspace_;
+      own_qspace = false;
+   }
+   vdim = (vdim_ < 0) ? vdim : vdim_;
+   NewDataAndSize(qf_data, vdim*qspace->GetSize());
+}
+
+void QuadratureFunction::Save(std::ostream &os) const
+{
+   GetSpace()->Save(os);
+   os << "VDim: " << vdim << '\n'
+      << '\n';
+   Vector::Print(os, vdim);
+   os.flush();
+}
+
+void QuadratureFunction::ProjectGridFunction(const GridFunction &gf)
+{
+   SetVDim(gf.VectorDim());
+
+   if (auto *qs_elem = dynamic_cast<QuadratureSpace*>(qspace))
+   {
+      const FiniteElementSpace &gf_fes = *gf.FESpace();
+      const bool use_tensor_products = UsesTensorBasis(gf_fes);
+      const ElementDofOrdering ordering = use_tensor_products ?
+                                          ElementDofOrdering::LEXICOGRAPHIC :
+                                          ElementDofOrdering::NATIVE;
+
+      // Use element restriction to go from L-vector to E-vector
+      const Operator *R = gf_fes.GetElementRestriction(ordering);
+      Vector e_vec(R->Height());
+      R->Mult(gf, e_vec);
+
+      // Use quadrature interpolator to go from E-vector to Q-vector
+      const QuadratureInterpolator *qi = gf_fes.GetQuadratureInterpolator(*qs_elem);
+      qi->SetOutputLayout(QVectorLayout::byVDIM);
+      qi->DisableTensorProducts(!use_tensor_products);
+      qi->Values(e_vec, *this);
+   }
+   else if (auto *qs_face = dynamic_cast<FaceQuadratureSpace*>(qspace))
+   {
+      const FiniteElementSpace &gf_fes = *gf.FESpace();
+      const bool use_tensor_products = UsesTensorBasis(gf_fes);
+      const ElementDofOrdering ordering = use_tensor_products ?
+                                          ElementDofOrdering::LEXICOGRAPHIC :
+                                          ElementDofOrdering::NATIVE;
+
+      const FaceType face_type = qs_face->GetFaceType();
+
+      // Use element restriction to go from L-vector to E-vector
+      const Operator *R = gf_fes.GetFaceRestriction(
+                             ordering, face_type, L2FaceValues::SingleValued);
+      Vector e_vec(R->Height());
+      R->Mult(gf, e_vec);
+
+      // Use quadrature interpolator to go from E-vector to Q-vector
+      const FaceQuadratureInterpolator *qi =
+         gf_fes.GetFaceQuadratureInterpolator(qspace->GetIntRule(0), face_type);
+      qi->SetOutputLayout(QVectorLayout::byVDIM);
+      qi->DisableTensorProducts(!use_tensor_products);
+      qi->Values(e_vec, *this);
+   }
+   else
+   {
+      // This branch should be unreachable
+      MFEM_ABORT("Unsupported case.");
+   }
+}
+
+std::ostream &operator<<(std::ostream &os, const QuadratureFunction &qf)
+{
+   qf.Save(os);
+   return os;
+}
+
+void QuadratureFunction::SaveVTU(std::ostream &os, VTKFormat format,
+                                 int compression_level) const
+{
+   os << R"(<VTKFile type="UnstructuredGrid" version="0.1")";
+   if (compression_level != 0)
+   {
+      os << R"( compressor="vtkZLibDataCompressor")";
+   }
+   os << " byte_order=\"" << VTKByteOrder() << "\">\n";
+   os << "<UnstructuredGrid>\n";
+
+   const char *fmt_str = (format == VTKFormat::ASCII) ? "ascii" : "binary";
+   const char *type_str = (format != VTKFormat::BINARY32) ? "Float64" : "Float32";
+   std::vector<char> buf;
+
+   Mesh &mesh = *qspace->GetMesh();
+
+   int np = qspace->GetSize();
+   int ne = mesh.GetNE();
+   int sdim = mesh.SpaceDimension();
+
+   // For quadrature functions, each point is a vertex cell, so number of cells
+   // is equal to number of points
+   os << "<Piece NumberOfPoints=\"" << np
+      << "\" NumberOfCells=\"" << np << "\">\n";
+
+   // print out the points
+   os << "<Points>\n";
+   os << "<DataArray type=\"" << type_str
+      << "\" NumberOfComponents=\"3\" format=\"" << fmt_str << "\">\n";
+
+   Vector pt(sdim);
+   for (int i = 0; i < ne; i++)
+   {
+      ElementTransformation &T = *mesh.GetElementTransformation(i);
+      const IntegrationRule &ir = GetIntRule(i);
+      for (int j = 0; j < ir.Size(); j++)
+      {
+         T.Transform(ir[j], pt);
+         WriteBinaryOrASCII(os, buf, pt[0], " ", format);
+         if (sdim > 1) { WriteBinaryOrASCII(os, buf, pt[1], " ", format); }
+         else { WriteBinaryOrASCII(os, buf, 0.0, " ", format); }
+         if (sdim > 2) { WriteBinaryOrASCII(os, buf, pt[2], "", format); }
+         else { WriteBinaryOrASCII(os, buf, 0.0, "", format); }
+         if (format == VTKFormat::ASCII) { os << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(os, buf, compression_level);
+   }
+   os << "</DataArray>\n";
+   os << "</Points>\n";
+
+   // Write cells (each cell is just a vertex)
+   os << "<Cells>\n";
+   // Connectivity
+   os << R"(<DataArray type="Int32" Name="connectivity" format=")"
+      << fmt_str << "\">\n";
+
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(os, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(os, buf, compression_level);
+   }
+   os << "</DataArray>\n";
+   // Offsets
+   os << R"(<DataArray type="Int32" Name="offsets" format=")"
+      << fmt_str << "\">\n";
+   for (int i=0; i<np; ++i) { WriteBinaryOrASCII(os, buf, i, "\n", format); }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(os, buf, compression_level);
+   }
+   os << "</DataArray>\n";
+   // Types
+   os << R"(<DataArray type="UInt8" Name="types" format=")"
+      << fmt_str << "\">\n";
+   for (int i = 0; i < np; i++)
+   {
+      uint8_t vtk_cell_type = VTKGeometry::POINT;
+      WriteBinaryOrASCII(os, buf, vtk_cell_type, "\n", format);
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(os, buf, compression_level);
+   }
+   os << "</DataArray>\n";
+   os << "</Cells>\n";
+
+   os << "<PointData>\n";
+   os << "<DataArray type=\"" << type_str << "\" Name=\"u\" format=\""
+      << fmt_str << "\" NumberOfComponents=\"" << vdim << "\">\n";
+   for (int i = 0; i < ne; i++)
+   {
+      DenseMatrix vals;
+      GetValues(i, vals);
+      for (int j = 0; j < vals.Size(); ++j)
+      {
+         for (int vd = 0; vd < vdim; ++vd)
+         {
+            WriteBinaryOrASCII(os, buf, vals(vd, j), " ", format);
+         }
+         if (format == VTKFormat::ASCII) { os << '\n'; }
+      }
+   }
+   if (format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(os, buf, compression_level);
+   }
+   os << "</DataArray>\n";
+   os << "</PointData>\n";
+
+   os << "</Piece>\n";
+   os << "</UnstructuredGrid>\n";
+   os << "</VTKFile>" << std::endl;
+}
+
+void QuadratureFunction::SaveVTU(const std::string &filename, VTKFormat format,
+                                 int compression_level) const
+{
+   std::ofstream f(filename + ".vtu");
+   SaveVTU(f, format, compression_level);
+}
+
+}

fem/qfunction.hpp

@@ -0,0 +1,267 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_QFUNCTION
+#define MFEM_QFUNCTION
+
+#include "../config/config.hpp"
+#include "qspace.hpp"
+#include "gridfunc.hpp"
+
+namespace mfem
+{
+
+/// Represents values or vectors of values at quadrature points on a mesh.
+class QuadratureFunction : public Vector
+{
+protected:
+   QuadratureSpaceBase *qspace; ///< Associated QuadratureSpaceBase object.
+   bool own_qspace; ///< Does this own the associated QuadratureSpaceBase?
+   int vdim; ///< Vector dimension.
+
+public:
+   /// Default constructor, results in an empty vector.
+   QuadratureFunction() : qspace(nullptr), own_qspace(false), vdim(0) { }
+
+   /// Create a QuadratureFunction based on the given QuadratureSpaceBase.
+   /** The QuadratureFunction does not assume ownership of the
+       QuadratureSpaceBase.
+       @note The Vector data is not initialized. */
+   QuadratureFunction(QuadratureSpaceBase &qspace_, int vdim_ = 1)
+      : Vector(vdim_*qspace_.GetSize()),
+        qspace(&qspace_), own_qspace(false), vdim(vdim_)
+   { }
+
+   /// Create a QuadratureFunction based on the given QuadratureSpaceBase.
+   /** The QuadratureFunction does not assume ownership of the
+       QuadratureSpaceBase.
+       @warning @a qspace_ may not be NULL. */
+   QuadratureFunction(QuadratureSpaceBase *qspace_, int vdim_ = 1)
+      : QuadratureFunction(*qspace_, vdim_) { }
+
+   /** @brief Copy constructor. The QuadratureSpace ownership flag, #own_qspace,
+       in the new object is set to false. */
+   QuadratureFunction(const QuadratureFunction &orig)
+      : QuadratureFunction(*orig.qspace, orig.vdim)
+   {
+      Vector::operator=(orig);
+   }
+
+   /// Read a QuadratureFunction from the stream @a in.
+   /** The QuadratureFunction assumes ownership of the read QuadratureSpace. */
+   QuadratureFunction(Mesh *mesh, std::istream &in);
+
+   /// Get the vector dimension.
+   int GetVDim() const { return vdim; }
+
+   /// Set the vector dimension, updating the size by calling Vector::SetSize().
+   void SetVDim(int vdim_)
+   { vdim = vdim_; SetSize(vdim*qspace->GetSize()); }
+
+   /// Get the associated QuadratureSpaceBase object.
+   QuadratureSpaceBase *GetSpace() { return qspace; }
+
+   /// Get the associated QuadratureSpaceBase object (const version).
+   const QuadratureSpaceBase *GetSpace() const { return qspace; }
+
+   /// Change the QuadratureSpaceBase and optionally the vector dimension.
+   /** If the new QuadratureSpaceBase is different from the current one, the
+       QuadratureFunction will not assume ownership of the new space; otherwise,
+       the ownership flag remains the same.
+
+       If the new vector dimension @a vdim_ < 0, the vector dimension remains
+       the same.
+
+       The data size is updated by calling Vector::SetSize(). */
+   inline void SetSpace(QuadratureSpaceBase *qspace_, int vdim_ = -1);
+
+   /** @brief Change the QuadratureSpaceBase, the data array, and optionally the
+       vector dimension. */
+   /** If the new QuadratureSpaceBase is different from the current one, the
+       QuadratureFunction will not assume ownership of the new space; otherwise,
+       the ownership flag remains the same.
+
+       If the new vector dimension @a vdim_ < 0, the vector dimension remains
+       the same.
+
+       The data array is replaced by calling Vector::NewDataAndSize(). */
+   inline void SetSpace(QuadratureSpaceBase *qspace_, double *qf_data,
+                        int vdim_ = -1);
+
+   /// Get the QuadratureSpaceBase ownership flag.
+   bool OwnsSpace() { return own_qspace; }
+
+   /// Set the QuadratureSpaceBase ownership flag.
+   void SetOwnsSpace(bool own) { own_qspace = own; }
+
+   /// Set this equal to a constant value.
+   QuadratureFunction &operator=(double value);
+
+   /// Copy the data from @a v.
+   /** The size of @a v must be equal to the size of the associated
+       QuadratureSpaceBase #qspace times the QuadratureFunction vector
+       dimension i.e. QuadratureFunction::Size(). */
+   QuadratureFunction &operator=(const Vector &v);
+
+   /// Evaluate a grid function at each quadrature point.
+   void ProjectGridFunction(const GridFunction &gf);
+
+   /// Return all values associated with mesh element @a idx in a Vector.
+   /** The result is stored in the Vector @a values as a reference to the
+       global values.
+
+       Inside the Vector @a values, the index `i+vdim*j` corresponds to the
+       `i`-th vector component at the `j`-th quadrature point.
+    */
+   inline void GetValues(int idx, Vector &values);
+
+   /// Return all values associated with mesh element @a idx in a Vector.
+   /** The result is stored in the Vector @a values as a copy of the
+       global values.
+
+       Inside the Vector @a values, the index `i+vdim*j` corresponds to the
+       `i`-th vector component at the `j`-th quadrature point.
+    */
+   inline void GetValues(int idx, Vector &values) const;
+
+   /// Return the quadrature function values at an integration point.
+   /** The result is stored in the Vector @a values as a reference to the
+       global values. */
+   inline void GetValues(int idx, const int ip_num, Vector &values);
+
+   /// Return the quadrature function values at an integration point.
+   /** The result is stored in the Vector @a values as a copy to the
+       global values. */
+   inline void GetValues(int idx, const int ip_num, Vector &values) const;
+
+   /// Return all values associated with mesh element @a idx in a DenseMatrix.
+   /** The result is stored in the DenseMatrix @a values as a reference to the
+       global values.
+
+       Inside the DenseMatrix @a values, the `(i,j)` entry corresponds to the
+       `i`-th vector component at the `j`-th quadrature point.
+    */
+   inline void GetValues(int idx, DenseMatrix &values);
+
+   /// Return all values associated with mesh element @a idx in a const DenseMatrix.
+   /** The result is stored in the DenseMatrix @a values as a copy of the
+       global values.
+
+       Inside the DenseMatrix @a values, the `(i,j)` entry corresponds to the
+       `i`-th vector component at the `j`-th quadrature point.
+    */
+   inline void GetValues(int idx, DenseMatrix &values) const;
+
+   /// Get the IntegrationRule associated with entity (element or face) @a idx.
+   const IntegrationRule &GetIntRule(int idx) const
+   { return GetSpace()->GetIntRule(idx); }
+
+   /// Write the QuadratureFunction to the stream @a out.
+   void Save(std::ostream &out) const;
+
+   /// @brief Write the QuadratureFunction to @a out in VTU (ParaView) format.
+   ///
+   /// The data will be uncompressed if @a compression_level is zero, or if the
+   /// format is VTKFormat::ASCII. Otherwise, zlib compression will be used for
+   /// binary data.
+   void SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
+
+   /// @brief Save the QuadratureFunction to a VTU (ParaView) file.
+   ///
+   /// The extension ".vtu" will be appended to @a filename.
+   /// @sa SaveVTU(std::ostream &out, VTKFormat format=VTKFormat::ASCII,
+   ///             int compression_level=0)
+   void SaveVTU(const std::string &filename, VTKFormat format=VTKFormat::ASCII,
+                int compression_level=0) const;
+
+   virtual ~QuadratureFunction()
+   {
+      if (own_qspace) { delete qspace; }
+   }
+};
+
+// Inline methods
+
+inline void QuadratureFunction::GetValues(
+   int idx, Vector &values)
+{
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   values.MakeRef(*this, vdim*s_offset, vdim*sl_size);
+}
+
+inline void QuadratureFunction::GetValues(
+   int idx, Vector &values) const
+{
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   values.SetSize(vdim*sl_size);
+   values.HostWrite();
+   const double *q = HostRead() + vdim*s_offset;
+   for (int i = 0; i<values.Size(); i++)
+   {
+      values(i) = *(q++);
+   }
+}
+
+inline void QuadratureFunction::GetValues(
+   int idx, const int ip_num, Vector &values)
+{
+   const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim;
+   values.MakeRef(*this, s_offset, vdim);
+}
+
+inline void QuadratureFunction::GetValues(
+   int idx, const int ip_num, Vector &values) const
+{
+   const int s_offset = qspace->offsets[idx] * vdim + ip_num * vdim;
+   values.SetSize(vdim);
+   values.HostWrite();
+   const double *q = HostRead() + s_offset;
+   for (int i = 0; i < values.Size(); i++)
+   {
+      values(i) = *(q++);
+   }
+}
+
+inline void QuadratureFunction::GetValues(
+   int idx, DenseMatrix &values)
+{
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   // Make the values matrix memory an alias of the quadrature function memory
+   Memory<double> &values_mem = values.GetMemory();
+   values_mem.Delete();
+   values_mem.MakeAlias(GetMemory(), vdim*s_offset, vdim*sl_size);
+   values.SetSize(vdim, sl_size);
+}
+
+inline void QuadratureFunction::GetValues(
+   int idx, DenseMatrix &values) const
+{
+   const int s_offset = qspace->offsets[idx];
+   const int sl_size = qspace->offsets[idx+1] - s_offset;
+   values.SetSize(vdim, sl_size);
+   values.HostWrite();
+   const double *q = HostRead() + vdim*s_offset;
+   for (int j = 0; j<sl_size; j++)
+   {
+      for (int i = 0; i<vdim; i++)
+      {
+         values(i,j) = *(q++);
+      }
+   }
+}
+
+} // namespace mfem
+
+#endif

fem/qinterp/det.cpp

@@ -239,7 +239,7 @@ void TensorDeterminants(const int NE,
          {
             constexpr int MD = 6;
             constexpr int MQ = 6;
-            // Highest orders that fit in shared mememory
+            // Highest orders that fit in shared memory
             if (D1D <= MD && Q1D <= MQ)
             { return Det3D<0,0,MD,MQ>(NE,B,G,X,Y,vdim,D1D,Q1D); }
             // Last fall-back will use global memory

fem/qspace.cpp

@@ -0,0 +1,171 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "qspace.hpp"
+
+namespace mfem
+{
+
+QuadratureSpaceBase::QuadratureSpaceBase(Mesh &mesh_, Geometry::Type geom,
+                                         const IntegrationRule &ir)
+   : mesh(mesh_)
+{
+   for (int g = 0; g < Geometry::NumGeom; g++)
+   {
+      int_rule[g] = NULL;
+   }
+   int_rule[geom] = &ir;
+}
+
+void QuadratureSpaceBase::ConstructIntRules(int dim)
+{
+   Array<Geometry::Type> geoms;
+   mesh.GetGeometries(dim, geoms);
+   for (Geometry::Type geom : geoms)
+   {
+      int_rule[geom] = &IntRules.Get(geom, order);
+   }
+}
+
+void QuadratureSpace::ConstructOffsets()
+{
+   const int num_elem = mesh.GetNE();
+   offsets.SetSize(num_elem + 1);
+   int offset = 0;
+   for (int i = 0; i < num_elem; i++)
+   {
+      offsets[i] = offset;
+      int geom = mesh.GetElementBaseGeometry(i);
+      MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule.");
+      offset += int_rule[geom]->GetNPoints();
+   }
+   offsets[num_elem] = size = offset;
+}
+
+void QuadratureSpace::Construct()
+{
+   ConstructIntRules(mesh.Dimension());
+   ConstructOffsets();
+}
+
+QuadratureSpace::QuadratureSpace(Mesh *mesh_, std::istream &in)
+   : QuadratureSpaceBase(*mesh_)
+{
+   const char *msg = "invalid input stream";
+   std::string ident;
+
+   in >> ident; MFEM_VERIFY(ident == "QuadratureSpace", msg);
+   in >> ident; MFEM_VERIFY(ident == "Type:", msg);
+   in >> ident;
+   if (ident == "default_quadrature")
+   {
+      in >> ident; MFEM_VERIFY(ident == "Order:", msg);
+      in >> order;
+   }
+   else
+   {
+      MFEM_ABORT("unknown QuadratureSpace type: " << ident);
+      return;
+   }
+
+   Construct();
+}
+
+QuadratureSpace::QuadratureSpace(Mesh &mesh_, const IntegrationRule &ir)
+   : QuadratureSpaceBase(mesh_, mesh_.GetElementGeometry(0), ir)
+{
+   MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension()) == 1,
+               "Constructor not valid for mixed meshes");
+   ConstructOffsets();
+}
+
+void QuadratureSpace::Save(std::ostream &os) const
+{
+   os << "QuadratureSpace\n"
+      << "Type: default_quadrature\n"
+      << "Order: " << order << '\n';
+}
+
+FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, int order_,
+                                         FaceType face_type_)
+   : QuadratureSpaceBase(mesh_, order_),
+     face_type(face_type_),
+     num_faces(mesh.GetNFbyType(face_type))
+{
+   Construct();
+}
+
+FaceQuadratureSpace::FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,
+                                         FaceType face_type_)
+   : QuadratureSpaceBase(mesh_, mesh_.GetFaceGeometry(0), ir),
+     face_type(face_type_),
+     num_faces(mesh.GetNFbyType(face_type))
+{
+   MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension() - 1) == 1,
+               "Constructor not valid for mixed meshes");
+   ConstructOffsets();
+}
+
+void FaceQuadratureSpace::ConstructOffsets()
+{
+   face_indices.SetSize(num_faces);
+   offsets.SetSize(num_faces + 1);
+   int offset = 0;
+   int f_idx = 0;
+   for (int i = 0; i < mesh.GetNumFacesWithGhost(); i++)
+   {
+      const Mesh::FaceInformation face = mesh.GetFaceInformation(i);
+      if (face.IsNonconformingCoarse() || !face.IsOfFaceType(face_type))
+      {
+         continue;
+      }
+      face_indices[f_idx] = i;
+      offsets[f_idx] = offset;
+      Geometry::Type geom = mesh.GetFaceGeometry(i);
+      MFEM_ASSERT(int_rule[geom] != NULL, "Missing integration rule");
+      offset += int_rule[geom]->GetNPoints();
+
+      f_idx++;
+   }
+   offsets[num_faces] = size = offset;
+}
+
+void FaceQuadratureSpace::Construct()
+{
+   ConstructIntRules(mesh.Dimension() - 1);
+   ConstructOffsets();
+}
+
+int FaceQuadratureSpace::GetPermutedIndex(int idx, int iq) const
+{
+   const int f_idx = face_indices[idx];
+   if (Geometry::IsTensorProduct(GetGeometry(idx)))
+   {
+      const int dim = mesh.Dimension();
+      const IntegrationRule &ir = GetIntRule(idx);
+      const int q1d = (int)floor(pow(ir.GetNPoints(), 1.0/(dim-1)) + 0.5);
+      const Mesh::FaceInformation face = mesh.GetFaceInformation(f_idx);
+      return ToLexOrdering(dim, face.element[0].local_face_id, q1d, iq);
+   }
+   else
+   {
+      return iq;
+   }
+}
+
+void FaceQuadratureSpace::Save(std::ostream &os) const
+{
+   os << "FaceQuadratureSpace\n"
+      << "Type: default_quadrature\n"
+      << "Order: " << order << '\n';
+}
+
+} // namespace mfem

fem/qspace.hpp

@@ -0,0 +1,189 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_QSPACE
+#define MFEM_QSPACE
+
+#include "../config/config.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+/// Abstract base class for QuadratureSpace and FaceQuadratureSpace.
+/** This class represents the storage layout for QuadratureFunction%s, that may
+    be defined either on mesh elements or mesh faces. */
+class QuadratureSpaceBase
+{
+protected:
+   friend class QuadratureFunction; // Uses the offsets.
+
+   Mesh &mesh; ///< The underlying mesh.
+   int order; ///< The order of integration rule.
+   int size; ///< Total number of quadrature points.
+
+   /// @brief Entity quadrature point offset array, of size num_entities + 1.
+   ///
+   /// The quadrature point values for entity i are stored in the indices between
+   /// offsets[i] and offsets[i+1].
+   Array<int> offsets;
+   /// The quadrature rules used for each geometry type.
+   const IntegrationRule *int_rule[Geometry::NumGeom];
+
+   /// Protected constructor. Used by derived classes.
+   QuadratureSpaceBase(Mesh &mesh_, int order_ = 0)
+      : mesh(mesh_), order(order_) { }
+
+   /// Protected constructor. Used by derived classes.
+   QuadratureSpaceBase(Mesh &mesh_, Geometry::Type geom,
+                       const IntegrationRule &ir);
+
+   /// Fill the @ref int_rule array for each geometry type using @ref order.
+   void ConstructIntRules(int dim);
+
+public:
+   /// Return the total number of quadrature points.
+   int GetSize() const { return size; }
+
+   /// Return the order of the quadrature rule(s) used by all elements.
+   int GetOrder() const { return order; }
+
+   /// Return the number of entities.
+   int GetNE() const { return offsets.Size() - 1; }
+
+   /// Returns the mesh.
+   inline Mesh *GetMesh() const { return &mesh; }
+
+   /// Get the (element or face) transformation of entity @a idx.
+   virtual ElementTransformation *GetTransformation(int idx) = 0;
+
+   /// Return the geometry type of entity (element or face) @a idx.
+   virtual Geometry::Type GetGeometry(int idx) const = 0;
+
+   /// Return the IntegrationRule associated with entity @a idx.
+   const IntegrationRule &GetIntRule(int idx) const
+   { return *int_rule[GetGeometry(idx)]; }
+
+   /// @brief Returns the permuted index of the @a iq quadrature point in entity
+   /// @a idx.
+   ///
+   /// For tensor-product faces, returns the lexicographic index of the
+   /// quadrature point, oriented relative to "element 1". For QuadratureSpace%s
+   /// defined on elements (not faces), the permutation is trivial, and this
+   /// returns @a iq.
+   virtual int GetPermutedIndex(int idx, int iq) const = 0;
+
+   /// Write the QuadratureSpace to the stream @a out.
+   virtual void Save(std::ostream &out) const = 0;
+
+   virtual ~QuadratureSpaceBase() { }
+};
+
+/// Class representing the storage layout of a QuadratureFunction.
+/** Multiple QuadratureFunction%s can share the same QuadratureSpace. */
+class QuadratureSpace : public QuadratureSpaceBase
+{
+protected:
+   void ConstructOffsets();
+   void Construct();
+public:
+   /// Create a QuadratureSpace based on the global rules from #IntRules.
+   QuadratureSpace(Mesh *mesh_, int order_)
+      : QuadratureSpaceBase(*mesh_, order_) { Construct(); }
+
+   /// @brief Create a QuadratureSpace with an IntegrationRule, valid only when
+   /// the mesh has one element type.
+   QuadratureSpace(Mesh &mesh_, const IntegrationRule &ir);
+
+   /// Read a QuadratureSpace from the stream @a in.
+   QuadratureSpace(Mesh *mesh_, std::istream &in);
+
+   /// Returns number of elements in the mesh.
+   inline int GetNE() const { return mesh.GetNE(); }
+
+   /// Returns the element transformation of element @a idx.
+   ElementTransformation *GetTransformation(int idx) override
+   { return mesh.GetElementTransformation(idx); }
+
+   /// Returns the geometry type of element @a idx.
+   Geometry::Type GetGeometry(int idx) const override
+   { return mesh.GetElementGeometry(idx); }
+
+   /// Get the IntegrationRule associated with mesh element @a idx.
+   const IntegrationRule &GetElementIntRule(int idx) const
+   { return *int_rule[mesh.GetElementBaseGeometry(idx)]; }
+
+   /// @brief Returns the permuted index of the @a iq quadrature point in entity
+   /// @a idx.
+   ///
+   /// The member function QuadratureSpace::GetPermutedIndex always returns @a
+   /// iq, the permutation is only nontrivial for FaceQuadratureSpace.
+   int GetPermutedIndex(int idx, int iq) const override { return iq; }
+
+   /// Write the QuadratureSpace to the stream @a out.
+   void Save(std::ostream &out) const override;
+};
+
+/// Class representing the storage layout of a FaceQuadratureFunction.
+/** FaceQuadratureSpace is defined on either the interior or boundary faces
+    of a mesh, depending on the provided FaceType. */
+class FaceQuadratureSpace : public QuadratureSpaceBase
+{
+   FaceType face_type; ///< Is the space defined on interior or boundary faces?
+   const int num_faces; ///< Number of faces.
+
+   /// Map from boundary or interior face indices to mesh face indices.
+   Array<int> face_indices;
+
+   void ConstructOffsets();
+   void Construct();
+
+public:
+   /// Create a FaceQuadratureSpace based on the global rules from #IntRules.
+   FaceQuadratureSpace(Mesh &mesh_, int order_, FaceType face_type_);
+
+   /// @brief Create a FaceQuadratureSpace with an IntegrationRule, valid only
+   /// when the mesh has one type of face geometry.
+   FaceQuadratureSpace(Mesh &mesh_, const IntegrationRule &ir,
+                       FaceType face_type_);
+
+   /// Returns number of faces in the mesh.
+   inline int GetNumFaces() const { return num_faces; }
+
+   /// Returns the face type (boundary or interior).
+   FaceType GetFaceType() const { return face_type; }
+
+   /// Returns the face transformation of face @a idx.
+   ElementTransformation *GetTransformation(int idx) override
+   { return mesh.GetFaceTransformation(face_indices[idx]); }
+
+   /// Returns the geometry type of face @a idx.
+   Geometry::Type GetGeometry(int idx) const override
+   { return mesh.GetFaceGeometry(face_indices[idx]); }
+
+   /// Get the IntegrationRule associated with mesh element @a idx.
+   const IntegrationRule &GetFaceIntRule(int idx) const
+   { return *int_rule[GetGeometry(idx)]; }
+
+   /// @brief Returns the permuted index of the @a iq quadrature point in entity
+   /// @a idx.
+   ///
+   /// For tensor-product faces, returns the lexicographic index of the
+   /// quadrature point, oriented relative to "element 1".
+   int GetPermutedIndex(int idx, int iq) const override;
+
+   /// Write the FaceQuadratureSpace to the stream @a out.
+   void Save(std::ostream &out) const override;
+};
+
+}
+
+#endif

fem/quadinterpolator.cpp

@@ -11,6 +11,7 @@
 
 #include "quadinterpolator.hpp"
 #include "qinterp/dispatch.hpp"
+#include "qspace.hpp"
 #include "../general/forall.hpp"
 #include "../linalg/dtensor.hpp"
 #include "../linalg/kernels.hpp"

fem/quadinterpolator.hpp

@@ -17,13 +17,6 @@
 namespace mfem
 {
 
-/// Type describing possible layouts for Q-vectors.
-enum class QVectorLayout
-{
-   byNODES,  ///< NQPT x VDIM x NE (values) / NQPT x VDIM x DIM x NE (grads)
-   byVDIM    ///< VDIM x NQPT x NE (values) / VDIM x DIM x NQPT x NE (grads)
-};
-
 /** @brief A class that performs interpolation from an E-vector to quadrature
     point values and/or derivatives (Q-vectors). */
 /** An E-vector represents the element-wise discontinuous version of the FE

fem/quadinterpolator_face.cpp

@@ -68,7 +68,8 @@ static void GetSigns(const FiniteElementSpace &fes, const FaceType type,
 FaceQuadratureInterpolator::FaceQuadratureInterpolator(
    const FiniteElementSpace &fes,
    const IntegrationRule &ir, FaceType type_)
-   : type(type_), nf(fes.GetNFbyType(type)), signs(nf)
+   : type(type_), nf(fes.GetNFbyType(type)), signs(nf),
+     q_layout(QVectorLayout::byNODES)
 {
    fespace = &fes;
    IntRule = &ir;
@@ -93,6 +94,7 @@ template<const int T_VDIM, const int T_ND1D, const int T_NQ1D>
 void FaceQuadratureInterpolator::Eval2D(
    const int NF,
    const int vdim,
+   const QVectorLayout q_layout,
    const DofToQuad &maps,
    const Array<bool> &signs,
    const Vector &f_vec,
@@ -114,10 +116,14 @@ void FaceQuadratureInterpolator::Eval2D(
    auto G = Reshape(maps.G.Read(), NQ1D, ND1D);
    auto F = Reshape(f_vec.Read(), ND1D, VDIM, NF);
    auto sign = signs.Read();
-   auto val = Reshape(q_val.Write(), NQ1D, VDIM, NF);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ1D, VDIM, NF):
+              Reshape(q_val.Write(), VDIM, NQ1D, NF);
    // auto der = Reshape(q_der.Write(), NQ1D, VDIM, NF); // only tangential der
    auto det = Reshape(q_det.Write(), NQ1D, NF);
-   auto n   = Reshape(q_nor.Write(), NQ1D, VDIM, NF);
+   auto n   = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_nor.Write(), NQ1D, 2, NF):
+              Reshape(q_nor.Write(), 2, NQ1D, NF);
    MFEM_VERIFY(eval_flags | DERIVATIVES,
                "Derivatives on the faces are not yet supported.");
    // If Gauss-Lobatto
@@ -147,7 +153,11 @@ void FaceQuadratureInterpolator::Eval2D(
                const double b = B(q,d);
                for (int c = 0; c < VDIM; c++) { ed[c] += b*r_F[d][c]; }
             }
-            for (int c = 0; c < VDIM; c++) { val(q,c,f) = ed[c]; }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,f) = ed[c]; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,f) = ed[c]; }
+            }
          }
          if ((eval_flags & DERIVATIVES)
              || (eval_flags & DETERMINANTS)
@@ -176,8 +186,16 @@ void FaceQuadratureInterpolator::Eval2D(
                if (eval_flags & NORMALS)
                {
                   const double s = sign[f] ? -1.0 : 1.0;
-                  n(q,0,f) =  s*D[1]/norm;
-                  n(q,1,f) = -s*D[0]/norm;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     n(0,q,f) =  s*D[1]/norm;
+                     n(1,q,f) = -s*D[0]/norm;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     n(q,0,f) =  s*D[1]/norm;
+                     n(q,1,f) = -s*D[0]/norm;
+                  }
                }
             }
          }
@@ -189,6 +207,7 @@ template<const int T_VDIM, const int T_ND1D, const int T_NQ1D>
 void FaceQuadratureInterpolator::Eval3D(
    const int NF,
    const int vdim,
+   const QVectorLayout q_layout,
    const DofToQuad &maps,
    const Array<bool> &signs,
    const Vector &e_vec,
@@ -210,10 +229,14 @@ void FaceQuadratureInterpolator::Eval3D(
    auto G = Reshape(maps.G.Read(), NQ1D, ND1D);
    auto F = Reshape(e_vec.Read(), ND1D, ND1D, VDIM, NF);
    auto sign = signs.Read();
-   auto val = Reshape(q_val.Write(), NQ1D, NQ1D, VDIM, NF);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ1D, NQ1D, VDIM, NF):
+              Reshape(q_val.Write(), VDIM, NQ1D, NQ1D, NF);
    // auto der = Reshape(q_der.Write(), NQ1D, VDIM, 3, NF);
    auto det = Reshape(q_det.Write(), NQ1D, NQ1D, NF);
-   auto nor = Reshape(q_nor.Write(), NQ1D, NQ1D, 3, NF);
+   auto nor = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_nor.Write(), NQ1D, NQ1D, 3, NF):
+              Reshape(q_nor.Write(), 3, NQ1D, NQ1D, NF);
    MFEM_VERIFY(eval_flags | DERIVATIVES,
                "Derivatives on the faces are not yet supported.");
    MFEM_FORALL(f, NF,
@@ -266,7 +289,9 @@ void FaceQuadratureInterpolator::Eval3D(
                }
                for (int c = 0; c < VDIM; c++)
                {
-                  val(q1,q2,c,f) = BBu[q2][q1][c];
+                  const double v = BBu[q2][q1][c];
+                  if (q_layout == QVectorLayout::byVDIM)  { val(c,q1,q2,f) = v; }
+                  if (q_layout == QVectorLayout::byNODES) { val(q1,q2,c,f) = v; }
                }
             }
          }
@@ -342,9 +367,18 @@ void FaceQuadratureInterpolator::Eval3D(
                   if (eval_flags & DETERMINANTS) { det(q1,q2,f) = norm; }
                   if (eval_flags & NORMALS)
                   {
-                     nor(q1,q2,0,f) = n[0]/norm;
-                     nor(q1,q2,1,f) = n[1]/norm;
-                     nor(q1,q2,2,f) = n[2]/norm;
+                     if (q_layout == QVectorLayout::byVDIM)
+                     {
+                        nor(0,q1,q2,f) = n[0]/norm;
+                        nor(1,q1,q2,f) = n[1]/norm;
+                        nor(2,q1,q2,f) = n[2]/norm;
+                     }
+                     if (q_layout == QVectorLayout::byNODES)
+                     {
+                        nor(q1,q2,0,f) = n[0]/norm;
+                        nor(q1,q2,1,f) = n[1]/norm;
+                        nor(q1,q2,2,f) = n[2]/norm;
+                     }
                   }
                }
             }
@@ -357,6 +391,7 @@ template<const int T_VDIM, const int T_ND1D, const int T_NQ1D>
 void FaceQuadratureInterpolator::SmemEval3D(
    const int NF,
    const int vdim,
+   const QVectorLayout q_layout,
    const DofToQuad &maps,
    const Array<bool> &signs,
    const Vector &e_vec,
@@ -379,10 +414,14 @@ void FaceQuadratureInterpolator::SmemEval3D(
    auto G = Reshape(maps.G.Read(), NQ1D, ND1D);
    auto F = Reshape(e_vec.Read(), ND1D, ND1D, VDIM, NF);
    auto sign = signs.Read();
-   auto val = Reshape(q_val.Write(), NQ1D, NQ1D, VDIM, NF);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ1D, NQ1D, VDIM, NF):
+              Reshape(q_val.Write(), VDIM, NQ1D, NQ1D, NF);
    // auto der = Reshape(q_der.Write(), NQ1D, VDIM, 3, NF);
    auto det = Reshape(q_det.Write(), NQ1D, NQ1D, NF);
-   auto nor = Reshape(q_nor.Write(), NQ1D, NQ1D, 3, NF);
+   auto nor = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_nor.Write(), NQ1D, NQ1D, 3, NF):
+              Reshape(q_nor.Write(), 3, NQ1D, NQ1D, NF);
    MFEM_VERIFY(eval_flags | DERIVATIVES,
                "Derivatives on the faces are not yet supported.");
 
@@ -439,7 +478,8 @@ void FaceQuadratureInterpolator::SmemEval3D(
                   {
                      v += B(q2,d2)*Bu[q1][d2][c];
                   }
-                  val(q1,q2,c,f) = v;
+                  if (q_layout == QVectorLayout::byVDIM)  { val(c,q1,q2,f) = v; }
+                  if (q_layout == QVectorLayout::byNODES) { val(q1,q2,c,f) = v; }
                }
             }
          }
@@ -519,9 +559,18 @@ void FaceQuadratureInterpolator::SmemEval3D(
 
                      if (eval_flags & NORMALS)
                      {
-                        nor(q1,q2,0,f) = n[0]/norm;
-                        nor(q1,q2,1,f) = n[1]/norm;
-                        nor(q1,q2,2,f) = n[2]/norm;
+                        if (q_layout == QVectorLayout::byVDIM)
+                        {
+                           nor(0,q1,q2,f) = n[0]/norm;
+                           nor(1,q1,q2,f) = n[1]/norm;
+                           nor(2,q1,q2,f) = n[2]/norm;
+                        }
+                        if (q_layout == QVectorLayout::byNODES)
+                        {
+                           nor(q1,q2,0,f) = n[0]/norm;
+                           nor(q1,q2,1,f) = n[1]/norm;
+                           nor(q1,q2,2,f) = n[2]/norm;
+                        }
                      }
                   }
                }
@@ -547,6 +596,7 @@ void FaceQuadratureInterpolator::Mult(
    void (*eval_func)(
       const int NF,
       const int vdim,
+      const QVectorLayout q_layout,
       const DofToQuad &maps,
       const Array<bool> &signs,
       const Vector &e_vec,
@@ -667,7 +717,7 @@ void FaceQuadratureInterpolator::Mult(
    }
    if (eval_func)
    {
-      eval_func(nf, vdim, maps, signs, e_vec,
+      eval_func(nf, vdim, q_layout, maps, signs, e_vec,
                 q_val, q_der, q_det, q_nor, eval_flags);
    }
    else

fem/quadinterpolator_face.hpp

@@ -33,6 +33,7 @@ protected:
 
    const FiniteElementSpace *fespace;  ///< Not owned
    const IntegrationRule *IntRule;     ///< Not owned
+   mutable QVectorLayout q_layout;     ///< Output Q-vector layout
 
    mutable bool use_tensor_products;
 
@@ -70,6 +71,16 @@ public:
    void DisableTensorProducts(bool disable = true) const
    { use_tensor_products = !disable; }
 
+   /** @brief Query the current output Q-vector layout. The default value is
+       QVectorLayout::byNODES. */
+   /** @sa SetOutputLayout(). */
+   QVectorLayout GetOutputLayout() const { return q_layout; }
+
+   /** @brief Set the desired output Q-vector layout. The default value is
+       QVectorLayout::byNODES. */
+   /** @sa GetOutputLayout(). */
+   void SetOutputLayout(QVectorLayout layout) const { q_layout = layout; }
+
    /// Interpolate the E-vector @a e_vec to quadrature points.
    /** The @a eval_flags are a bitwise mask of constants from the FaceEvalFlags
        enumeration. When the VALUES flag is set, the values at quadrature points
@@ -91,6 +102,7 @@ public:
    template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
    static void Eval2D(const int NF,
                       const int vdim,
+                      const QVectorLayout q_layout,
                       const DofToQuad &maps,
                       const Array<bool> &signs,
                       const Vector &e_vec,
@@ -104,6 +116,7 @@ public:
    template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
    static void Eval3D(const int NF,
                       const int vdim,
+                      const QVectorLayout q_layout,
                       const DofToQuad &maps,
                       const Array<bool> &signs,
                       const Vector &e_vec,
@@ -116,6 +129,7 @@ public:
    template<const int T_VDIM = 0, const int T_ND = 0, const int T_NQ = 0>
    static void SmemEval3D(const int NF,
                           const int vdim,
+                          const QVectorLayout q_layout,
                           const DofToQuad &maps,
                           const Array<bool> &signs,
                           const Vector &e_vec,

fem/restriction.cpp

@@ -147,7 +147,8 @@ void ElementRestriction::MultUnsigned(const Vector& x, Vector& y) const
    });
 }
 
-void ElementRestriction::MultTranspose(const Vector& x, Vector& y) const
+template <bool ADD>
+void ElementRestriction::AddMultTranspose(const Vector& x, Vector& y) const
 {
    // Assumes all elements have the same number of dofs
    const int nd = dof;
@@ -156,7 +157,7 @@ void ElementRestriction::MultTranspose(const Vector& x, Vector& y) const
    auto d_offsets = offsets.Read();
    auto d_indices = indices.Read();
    auto d_x = Reshape(x.Read(), nd, vd, ne);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   auto d_y = Reshape(ADD ? y.ReadWrite() : y.Write(), t?vd:ndofs, t?ndofs:vd);
    MFEM_FORALL(i, ndofs,
    {
       const int offset = d_offsets[i];
@@ -170,11 +171,24 @@ void ElementRestriction::MultTranspose(const Vector& x, Vector& y) const
             dof_value += ((d_indices[j] >= 0) ? d_x(idx_j % nd, c, idx_j / nd) :
                           -d_x(idx_j % nd, c, idx_j / nd));
          }
-         d_y(t?c:i,t?i:c) = dof_value;
+         if (ADD) { d_y(t?c:i,t?i:c) += dof_value; }
+         else { d_y(t?c:i,t?i:c) = dof_value; }
       }
    });
 }
 
+void ElementRestriction::MultTranspose(const Vector& x, Vector& y) const
+{
+   constexpr bool ADD = false;
+   AddMultTranspose<ADD>(x, y);
+}
+
+void ElementRestriction::AddMultTranspose(const Vector& x, Vector& y) const
+{
+   constexpr bool ADD = true;
+   AddMultTranspose<ADD>(x, y);
+}
+
 void ElementRestriction::MultTransposeUnsigned(const Vector& x, Vector& y) const
 {
    // Assumes all elements have the same number of dofs
@@ -506,13 +520,14 @@ void L2ElementRestriction::Mult(const Vector &x, Vector &y) const
    });
 }
 
-void L2ElementRestriction::MultTranspose(const Vector &x, Vector &y) const
+template <bool ADD>
+void L2ElementRestriction::AddMultTranspose(const Vector &x, Vector &y) const
 {
    const int nd = ndof;
    const int vd = vdim;
    const bool t = byvdim;
    auto d_x = Reshape(x.Read(), nd, vd, ne);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   auto d_y = Reshape(ADD ? y.ReadWrite() : y.Write(), t?vd:ndofs, t?ndofs:vd);
    MFEM_FORALL(i, ndofs,
    {
       const int idx = i;
@@ -520,11 +535,24 @@ void L2ElementRestriction::MultTranspose(const Vector &x, Vector &y) const
       const int e = idx / nd;
       for (int c = 0; c < vd; ++c)
       {
-         d_y(t?c:idx,t?idx:c) = d_x(dof, c, e);
+         if (ADD) { d_y(t?c:idx,t?idx:c) += d_x(dof, c, e); }
+         else { d_y(t?c:idx,t?idx:c) = d_x(dof, c, e); }
       }
    });
 }
 
+void L2ElementRestriction::MultTranspose(const Vector &x, Vector &y) const
+{
+   constexpr bool ADD = false;
+   AddMultTranspose<ADD>(x, y);
+}
+
+void L2ElementRestriction::AddMultTranspose(const Vector &x, Vector &y) const
+{
+   constexpr bool ADD = true;
+   AddMultTranspose<ADD>(x, y);
+}
+
 void L2ElementRestriction::FillI(SparseMatrix &mat) const
 {
    const int elem_dofs = ndof;
@@ -1774,6 +1802,31 @@ void InterpolationManager::LinearizeInterpolatorMapIntoVector()
    interp_map.clear();
 }
 
+void InterpolationManager::InitializeNCInterpConfig()
+{
+   // Count nonconforming faces
+   int num_nc_faces = 0;
+   for (int i = 0; i < interp_config.Size(); i++)
+   {
+      if ( interp_config[i].is_non_conforming )
+      {
+         num_nc_faces++;
+      }
+   }
+   // Set nc_interp_config
+   nc_interp_config.SetSize(num_nc_faces);
+   int nc_index = 0;
+   for (int i = 0; i < interp_config.Size(); i++)
+   {
+      auto & config = interp_config[i];
+      if ( config.is_non_conforming )
+      {
+         nc_interp_config[nc_index] = NCInterpConfig(i, config);
+         nc_index++;
+      }
+   }
+}
+
 NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
                                          const ElementDofOrdering ordering,
                                          const FaceType type,
@@ -1801,64 +1854,53 @@ NCL2FaceRestriction::NCL2FaceRestriction(const FiniteElementSpace &fes,
 
 void NCL2FaceRestriction::DoubleValuedNonconformingMult(
    const Vector& x, Vector& y) const
+{
+   DoubleValuedConformingMult(x, y);
+   DoubleValuedNonconformingInterpolation(y);
+}
+
+void NCL2FaceRestriction::DoubleValuedNonconformingInterpolation(
+   Vector& y) const
 {
    // Assumes all elements have the same number of dofs
    const int nface_dofs = face_dofs;
    const int vd = vdim;
-   const bool t = byvdim;
-   auto d_indices1 = scatter_indices1.Read();
-   auto d_indices2 = scatter_indices2.Read();
-   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_y = Reshape(y.Write(), nface_dofs, vd, 2, nf);
-   auto interp_config_ptr = interpolations.GetFaceInterpConfig().Read();
+   auto d_y = Reshape(y.ReadWrite(), nface_dofs, vd, 2, nf);
+   auto &nc_interp_config = interpolations.GetNCFaceInterpConfig();
+   const int num_nc_faces = nc_interp_config.Size();
+   if ( num_nc_faces == 0 ) { return; }
+   auto interp_config_ptr = nc_interp_config.Read();
    const int nc_size = interpolations.GetNumInterpolators();
    auto d_interp = Reshape(interpolations.GetInterpolators().Read(),
                            nface_dofs, nface_dofs, nc_size);
    static constexpr int max_nd = 16*16;
    MFEM_VERIFY(nface_dofs<=max_nd, "Too many degrees of freedom.");
-   MFEM_FORALL_3D(face, nf, nface_dofs, 1, 1,
+   MFEM_FORALL_3D(nc_face, num_nc_faces, nface_dofs, 1, 1,
    {
       MFEM_SHARED double dof_values[max_nd];
-      const InterpConfig conf = interp_config_ptr[face];
-      const int master_side = conf.master_side;
-      const int interp_index = conf.index;
-      for (int side = 0; side < 2; side++)
+      const NCInterpConfig conf = interp_config_ptr[nc_face];
+      if ( conf.is_non_conforming )
       {
-         if ( !conf.is_non_conforming || side!=master_side )
+         const int master_side = conf.master_side;
+         const int interp_index = conf.index;
+         const int face = conf.face_index;
+         for (int c = 0; c < vd; ++c)
          {
-            // No interpolation needed
             MFEM_FOREACH_THREAD(dof,x,nface_dofs)
             {
-               const int i = face*nface_dofs + dof;
-               const int idx = side==0 ? d_indices1[i] : d_indices2[i];
-               for (int c = 0; c < vd; ++c)
-               {
-                  d_y(dof, c, side, face) = d_x(t?c:idx, t?idx:c);
-               }
+               dof_values[dof] = d_y(dof, c, master_side, face);
             }
-         }
-         else // Interpolation from coarse to fine
-         {
-            for (int c = 0; c < vd; ++c)
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dof_out,x,nface_dofs)
             {
-               MFEM_FOREACH_THREAD(dof,x,nface_dofs)
+               double res = 0.0;
+               for (int dof_in = 0; dof_in<nface_dofs; dof_in++)
                {
-                  const int i = face*nface_dofs + dof;
-                  const int idx = side==0 ? d_indices1[i] : d_indices2[i];
-                  dof_values[dof] = d_x(t?c:idx, t?idx:c);
+                  res += d_interp(dof_out, dof_in, interp_index)*dof_values[dof_in];
                }
-               MFEM_SYNC_THREAD;
-               MFEM_FOREACH_THREAD(dof_out,x,nface_dofs)
-               {
-                  double res = 0.0;
-                  for (int dof_in = 0; dof_in<nface_dofs; dof_in++)
-                  {
-                     res += d_interp(dof_out, dof_in, interp_index)*dof_values[dof_in];
-                  }
-                  d_y(dof_out, c, side, face) = res;
-               }
-               MFEM_SYNC_THREAD;
+               d_y(dof_out, c, master_side, face) = res;
             }
+            MFEM_SYNC_THREAD;
          }
       }
    });
@@ -1892,23 +1934,34 @@ void NCL2FaceRestriction::SingleValuedNonconformingTransposeInterpolation(
       x_interp.SetSize(x.Size());
    }
    x_interp = x;
+   SingleValuedNonconformingTransposeInterpolationInPlace(x_interp);
+}
+
+
+void NCL2FaceRestriction::SingleValuedNonconformingTransposeInterpolationInPlace(
+   Vector& x) const
+{
    // Assumes all elements have the same number of dofs
    const int nface_dofs = face_dofs;
    const int vd = vdim;
    // Interpolation
    auto d_x = Reshape(x_interp.ReadWrite(), nface_dofs, vd, nf);
-   auto interp_config_ptr = interpolations.GetFaceInterpConfig().Read();
+   auto &nc_interp_config = interpolations.GetNCFaceInterpConfig();
+   const int num_nc_faces = nc_interp_config.Size();
+   if ( num_nc_faces == 0 ) { return; }
+   auto interp_config_ptr = nc_interp_config.Read();
    auto interpolators = interpolations.GetInterpolators().Read();
    const int nc_size = interpolations.GetNumInterpolators();
    auto d_interp = Reshape(interpolators, nface_dofs, nface_dofs, nc_size);
    static constexpr int max_nd = 16*16;
    MFEM_VERIFY(nface_dofs<=max_nd, "Too many degrees of freedom.");
-   MFEM_FORALL_3D(face, nf, nface_dofs, 1, 1,
+   MFEM_FORALL_3D(nc_face, num_nc_faces, nface_dofs, 1, 1,
    {
       MFEM_SHARED double dof_values[max_nd];
-      const InterpConfig conf = interp_config_ptr[face];
+      const NCInterpConfig conf = interp_config_ptr[nc_face];
       const int master_side = conf.master_side;
       const int interp_index = conf.index;
+      const int face = conf.face_index;
       if ( conf.is_non_conforming && master_side==0 )
       {
          // Interpolation from fine to coarse
@@ -1945,23 +1998,33 @@ void NCL2FaceRestriction::DoubleValuedNonconformingTransposeInterpolation(
       x_interp.SetSize(x.Size());
    }
    x_interp = x;
+   DoubleValuedNonconformingTransposeInterpolationInPlace(x_interp);
+}
+
+void NCL2FaceRestriction::DoubleValuedNonconformingTransposeInterpolationInPlace(
+   Vector& x) const
+{
    // Assumes all elements have the same number of dofs
    const int nface_dofs = face_dofs;
    const int vd = vdim;
    // Interpolation
-   auto d_x = Reshape(x_interp.ReadWrite(), nface_dofs, vd, 2, nf);
-   auto interp_config_ptr = interpolations.GetFaceInterpConfig().Read();
+   auto d_x = Reshape(x.ReadWrite(), nface_dofs, vd, 2, nf);
+   auto &nc_interp_config = interpolations.GetNCFaceInterpConfig();
+   const int num_nc_faces = nc_interp_config.Size();
+   if ( num_nc_faces == 0 ) { return; }
+   auto interp_config_ptr = nc_interp_config.Read();
    auto interpolators = interpolations.GetInterpolators().Read();
    const int nc_size = interpolations.GetNumInterpolators();
    auto d_interp = Reshape(interpolators, nface_dofs, nface_dofs, nc_size);
    static constexpr int max_nd = 16*16;
    MFEM_VERIFY(nface_dofs<=max_nd, "Too many degrees of freedom.");
-   MFEM_FORALL_3D(face, nf, nface_dofs, 1, 1,
+   MFEM_FORALL_3D(nc_face, num_nc_faces, nface_dofs, 1, 1,
    {
       MFEM_SHARED double dof_values[max_nd];
-      const InterpConfig conf = interp_config_ptr[face];
+      const NCInterpConfig conf = interp_config_ptr[nc_face];
       const int master_side = conf.master_side;
       const int interp_index = conf.index;
+      const int face = conf.face_index;
       if ( conf.is_non_conforming )
       {
          // Interpolation from fine to coarse
@@ -2016,6 +2079,35 @@ void NCL2FaceRestriction::AddMultTranspose(const Vector& x, Vector& y) const
    }
 }
 
+void NCL2FaceRestriction::AddMultTransposeInPlace(Vector& x, Vector& y) const
+{
+   if (nf==0) { return; }
+   if (type==FaceType::Interior)
+   {
+      if ( m==L2FaceValues::DoubleValued )
+      {
+         DoubleValuedNonconformingTransposeInterpolationInPlace(x);
+         DoubleValuedConformingAddMultTranspose(x, y);
+      }
+      else if ( m==L2FaceValues::SingleValued )
+      {
+         SingleValuedNonconformingTransposeInterpolationInPlace(x);
+         SingleValuedConformingAddMultTranspose(x, y);
+      }
+   }
+   else
+   {
+      if ( m==L2FaceValues::DoubleValued )
+      {
+         DoubleValuedConformingAddMultTranspose(x, y);
+      }
+      else if ( m==L2FaceValues::SingleValued )
+      {
+         SingleValuedConformingAddMultTranspose(x, y);
+      }
+   }
+}
+
 void NCL2FaceRestriction::FillI(SparseMatrix &mat,
                                 const bool keep_nbr_block) const
 {
@@ -2116,6 +2208,7 @@ void NCL2FaceRestriction::ComputeScatterIndicesAndOffsets(
 
    // Transform the interpolation matrix map into a contiguous memory structure.
    interpolations.LinearizeInterpolatorMapIntoVector();
+   interpolations.InitializeNCInterpConfig();
 }
 
 void NCL2FaceRestriction::ComputeGatherIndices(

fem/restriction.hpp

@@ -21,10 +21,19 @@ namespace mfem
 class FiniteElementSpace;
 enum class ElementDofOrdering;
 
+/// Abstract base class that defines an interface for element restrictions.
+class ElementRestrictionOperator : public Operator
+{
+public:
+   /// @brief Add the E-vector degrees of freedom @a x to the L-vector degrees
+   /// of freedom @a y.
+   virtual void AddMultTranspose(const Vector &x, Vector &y) const = 0;
+};
+
 /// Operator that converts FiniteElementSpace L-vectors to E-vectors.
 /** Objects of this type are typically created and owned by FiniteElementSpace
     objects, see FiniteElementSpace::GetElementRestriction(). */
-class ElementRestriction : public Operator
+class ElementRestriction : public ElementRestrictionOperator
 {
 private:
    /** This number defines the maximum number of elements any dof can belong to
@@ -58,6 +67,7 @@ public:
    ElementRestriction(const FiniteElementSpace&, ElementDofOrdering);
    void Mult(const Vector &x, Vector &y) const;
    void MultTranspose(const Vector &x, Vector &y) const;
+   void AddMultTranspose(const Vector &x, Vector &y) const;
 
    /// Compute Mult without applying signs based on DOF orientations.
    void MultUnsigned(const Vector &x, Vector &y) const;
@@ -85,6 +95,11 @@ public:
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
        pattern given by this ElementRestriction, and the values of ea_data. */
    void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
+   /// @private Not part of the public interface (device kernel limitation).
+   ///
+   /// Performs either MultTranspose or AddMultTranspose depending on the
+   /// boolean template parameter @a ADD.
+   template <bool ADD> void AddMultTranspose(const Vector &x, Vector &y) const;
 };
 
 /// Operator that converts L2 FiniteElementSpace L-vectors to E-vectors.
@@ -92,7 +107,7 @@ public:
     objects, see FiniteElementSpace::GetElementRestriction(). L-vectors
     corresponding to grid functions in L2 finite element spaces differ from
     E-vectors only in the ordering of the degrees of freedom. */
-class L2ElementRestriction : public Operator
+class L2ElementRestriction : public ElementRestrictionOperator
 {
    const int ne;
    const int vdim;
@@ -103,12 +118,18 @@ public:
    L2ElementRestriction(const FiniteElementSpace&);
    void Mult(const Vector &x, Vector &y) const;
    void MultTranspose(const Vector &x, Vector &y) const;
+   void AddMultTranspose(const Vector &x, Vector &y) const;
    /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
        given by this ElementRestriction. */
    void FillI(SparseMatrix &mat) const;
    /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
        pattern given by this L2FaceRestriction, and the values of ea_data. */
    void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
+   /// @private Not part of the public interface (device kernel limitation).
+   ///
+   /// Performs either MultTranspose or AddMultTranspose depending on the
+   /// boolean template parameter @a ADD.
+   template <bool ADD> void AddMultTranspose(const Vector &x, Vector &y) const;
 };
 
 /** An enum type to specify if only e1 value is requested (SingleValued) or both
@@ -162,6 +183,22 @@ public:
    */
    virtual void AddMultTranspose(const Vector &x, Vector &y) const = 0;
 
+   /** @brief Add the face degrees of freedom @a x to the element degrees of
+       freedom @a y. Perform the same computation as AddMultTranspose, but
+       @a x is invalid after calling this method.
+
+       @param[in,out]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+
+      @note This method is an optimization of AddMultTranspose where the @a x
+      Vector is used and modified to avoid memory allocation and memcpy.
+   */
+   virtual void AddMultTransposeInPlace(Vector &x, Vector &y) const
+   {
+      AddMultTranspose(x, y);
+   }
+
    /** @brief Set the face degrees of freedom in the element degrees of freedom
        @a y to the values given in @a x.
 
@@ -229,6 +266,8 @@ public:
                      ElementDofOrdering. */
    void Mult(const Vector &x, Vector &y) const override;
 
+   using FaceRestriction::AddMultTransposeInPlace;
+
    /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
        L-Vector.
 
@@ -358,6 +397,8 @@ public:
                      ElementDofOrdering. */
    void Mult(const Vector &x, Vector &y) const override;
 
+   using FaceRestriction::AddMultTranspose;
+
    /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
        L-Vector.
 
@@ -585,6 +626,39 @@ struct InterpConfig
    InterpConfig &operator=(const InterpConfig &rhs) = default;
 };
 
+/** This struct stores which side is the master nonconforming side and the
+    index of the interpolator, see InterpolationManager class below. */
+struct NCInterpConfig
+{
+   int face_index;
+   uint32_t is_non_conforming : 1;
+   uint32_t master_side : 1;
+   uint32_t index : 30;
+
+   // default constructor.
+   NCInterpConfig() = default;
+
+   // Non-conforming face
+   NCInterpConfig(int face_index, int master_side, int nc_index)
+      : face_index(face_index),
+        is_non_conforming(1),
+        master_side(master_side),
+        index(nc_index)
+   { }
+
+   // Non-conforming face
+   NCInterpConfig(int face_index, InterpConfig & config)
+      : face_index(face_index),
+        is_non_conforming(config.is_non_conforming),
+        master_side(config.master_side),
+        index(config.index)
+   { }
+
+   NCInterpConfig(const NCInterpConfig&) = default;
+
+   NCInterpConfig &operator=(const NCInterpConfig &rhs) = default;
+};
+
 /** @brief This class manages the storage and computation of the interpolations
     from master (coarse) face to slave (fine) face.
 */
@@ -594,6 +668,7 @@ protected:
    const FiniteElementSpace &fes;
    const ElementDofOrdering ordering;
    Array<InterpConfig> interp_config; // interpolator index for each face
+   Array<NCInterpConfig> nc_interp_config; // interpolator index for each ncface
    Vector interpolators; // face_dofs x face_dofs x num_interpolators
    int nc_cpt; // Counter for interpolators, and used as index.
 
@@ -639,6 +714,8 @@ public:
        structure. */
    void LinearizeInterpolatorMapIntoVector();
 
+   void InitializeNCInterpConfig();
+
    /// @brief Return the total number of interpolators.
    int GetNumInterpolators() const
    {
@@ -660,6 +737,14 @@ public:
       return interp_config;
    }
 
+   /** @brief Return an array containing the interpolation configuration for
+       each face registered with RegisterFaceConformingInterpolation and
+       RegisterFaceCoarseToFineInterpolation. */
+   const Array<NCInterpConfig>& GetNCFaceInterpConfig() const
+   {
+      return nc_interp_config;
+   }
+
 private:
    /** @brief Returns the interpolation operator from a master (coarse) face to
        a slave (fine) face.
@@ -749,6 +834,22 @@ public:
        @param[in,out] y The L-vector degrees of freedom. */
    void AddMultTranspose(const Vector &x, Vector &y) const override;
 
+   /** @brief Gather the degrees of freedom, i.e. goes from face E-Vector to
+       L-Vector.
+
+       @param[in,out]  x The face E-Vector degrees of freedom with the given format:
+                         if L2FacesValues::DoubleValued (face_dofs x vdim x 2 x nf),
+                         if L2FacesValues::SingleValued (face_dofs x vdim x nf),
+                         where nf is the number of interior or boundary faces
+                         requested by @a type in the constructor.
+                         The face_dofs should be ordered according to the given
+                         ElementDofOrdering
+       @param[in,out] y The L-vector degrees of freedom.
+
+      @note This method is an optimization of AddMultTranspose where the @a x
+      Vector is used and modified to avoid memory allocation and memcpy. */
+   void AddMultTransposeInPlace(Vector &x, Vector &y) const override;
+
    /** @brief Fill the I array of SparseMatrix corresponding to the sparsity
        pattern given by this NCL2FaceRestriction.
 
@@ -838,6 +939,14 @@ public:
                      ElementDofOrdering. */
    virtual void DoubleValuedNonconformingMult(const Vector& x, Vector& y) const;
 
+   /** @brief Apply a change of basis from coarse element basis to fine element
+       basis for the coarse face dofs.
+
+       @param[in,out] x The dofs vector that needs coarse dofs to be express in
+                        term of the fine basis.
+   */
+   void DoubleValuedNonconformingInterpolation(Vector& x) const;
+
    /** @brief Apply a change of basis from fine element basis to coarse element
        basis for the coarse face dofs. Should only be used when:
        L2FaceValues m == L2FaceValues::SingleValued
@@ -847,6 +956,15 @@ public:
    */
    void SingleValuedNonconformingTransposeInterpolation(const Vector& x) const;
 
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs. Should only be used when:
+       L2FaceValues m == L2FaceValues::SingleValued
+
+       @param[in,out] x The dofs vector that needs coarse dofs to be express in
+                        term of the coarse basis, the result is stored in x.
+   */
+   void SingleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
+
    /** @brief Apply a change of basis from fine element basis to coarse element
        basis for the coarse face dofs. Should only be used when:
        L2FaceValues m == L2FaceValues::DoubleValued
@@ -855,6 +973,16 @@ public:
                     of the coarse basis, the result is stored in x_interp.
    */
    void DoubleValuedNonconformingTransposeInterpolation(const Vector& x) const;
+
+   /** @brief Apply a change of basis from fine element basis to coarse element
+       basis for the coarse face dofs. Should only be used when:
+       L2FaceValues m == L2FaceValues::DoubleValued
+
+       @param[in,out] x The dofs vector that needs coarse dofs to be express in
+                        term of the coarse basis, the result is stored in
+                        x.
+   */
+   void DoubleValuedNonconformingTransposeInterpolationInPlace(Vector& x) const;
 };
 
 /** @brief Return the face map that extracts the degrees of freedom for the

fem/tmop.cpp

@@ -20,6 +20,16 @@ namespace mfem
 
 // Target-matrix optimization paradigm (TMOP) mesh quality metrics.
 
+double TMOP_Combo_QualityMetric::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   double metric = 0.;
+   for (int i = 0; i < tmop_q_arr.Size(); i++)
+   {
+      metric += wt_arr[i]*tmop_q_arr[i]->EvalWMatrixForm(Jpt);
+   }
+   return metric;
+}
+
 double TMOP_Combo_QualityMetric::EvalW(const DenseMatrix &Jpt) const
 {
    double metric = 0.;
@@ -232,6 +242,11 @@ double TMOP_Metric_aspratio3D::EvalW(const DenseMatrix &Jpt) const
           ) / 3.0;
 }
 
+double TMOP_Metric_002::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   return 0.5 * Jpt.FNorm2() / Jpt.Det() - 1.0;
+}
+
 double TMOP_Metric_002::EvalW(const DenseMatrix &Jpt) const
 {
    ie.SetJacobian(Jpt.GetData());
@@ -516,12 +531,23 @@ void TMOP_Metric_056::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_ddI2b(weight*(0.5 - 0.5/ie.Get_I2()), A.GetData());
 }
 
+double TMOP_Metric_058::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_58 = |J^t J|^2 / det(J)^2 - 2|J|^2 / det(J) + 2
+   DenseMatrix JtJ(2);
+   MultAAt(Jpt, JtJ);
+   JtJ.Transpose();
+   double det = Jpt.Det();
+
+   return JtJ.FNorm2()/(det*det) - 2*Jpt.FNorm2()/det + 2.0;
+}
+
 double TMOP_Metric_058::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_58 = I1b*(I1b - 2)
    ie.SetJacobian(Jpt.GetData());
    const double I1b = ie.Get_I1b();
-   return I1b*(I1b - 1.0);
+   return I1b*(I1b - 2.0);
 }
 
 void TMOP_Metric_058::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
@@ -668,8 +694,18 @@ void TMOP_Metric_252::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_ddI2b(weight*(c - 0.5*c*c), A.GetData());
 }
 
+double TMOP_Metric_301::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_301 = 1/3 |J| |J^-1| - 1.
+   ie.SetJacobian(Jpt.GetData());
+   DenseMatrix inv(3);
+   CalcInverse(Jpt, inv);
+   return Jpt.FNorm() * inv.FNorm() / 3.0 - 1.0;
+}
+
 double TMOP_Metric_301::EvalW(const DenseMatrix &Jpt) const
 {
+   // mu_301 = 1/3 sqrt(I1b * I2b) - 1
    ie.SetJacobian(Jpt.GetData());
    return std::sqrt(ie.Get_I1b()*ie.Get_I2b())/3. - 1.;
 }
@@ -718,6 +754,15 @@ void TMOP_Metric_301::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_TProd(a/(2*I1b_I2b), d_I1b_I2b_data, A.GetData());
 }
 
+double TMOP_Metric_302::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_301 = |J|^2 |J^{-1}|^2 / 9 - 1.
+   ie.SetJacobian(Jpt.GetData());
+   DenseMatrix inv(3);
+   CalcInverse(Jpt, inv);
+   return Jpt.FNorm2() * inv.FNorm2() / 9.0 - 1.0;
+}
+
 double TMOP_Metric_302::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_2 = |J|^2 |J^{-1}|^2 / 9 - 1
@@ -752,14 +797,24 @@ void TMOP_Metric_302::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_ddI1b(c1*ie.Get_I2b(), A.GetData());
 }
 
+double TMOP_Metric_303::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_303 = |J|^2 / 3 / det(J)^(2/3) - 1.
+   ie.SetJacobian(Jpt.GetData());
+   return Jpt.FNorm2() / 3.0 / pow(Jpt.Det(), 2.0 / 3.0) - 1.0;
+}
+
 double TMOP_Metric_303::EvalW(const DenseMatrix &Jpt) const
 {
+   // mu_303 = |J|^2 / 3 / det(J)^(2/3) - 1 = I1b/3 - 1.
    ie.SetJacobian(Jpt.GetData());
    return ie.Get_I1b()/3.0 - 1.0;
 }
 
 void TMOP_Metric_303::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
 {
+   // mu_304 = I1b/3 - 1.
+   // P      = dI1b/3.
    ie.SetJacobian(Jpt.GetData());
    P.Set(1./3., ie.Get_dI1b());
 }
@@ -769,11 +824,47 @@ void TMOP_Metric_303::AssembleH(const DenseMatrix &Jpt,
                                 const double weight,
                                 DenseMatrix &A) const
 {
+   // P  = dI1b/3.
+   // dP = ddI1b/3.
    ie.SetJacobian(Jpt.GetData());
    ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
    ie.Assemble_ddI1b(weight/3., A.GetData());
 }
 
+double TMOP_Metric_304::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_304 = |J|^3 / 3^(3/2) / det(J) - 1
+   const double fnorm = Jpt.FNorm();
+   return fnorm * fnorm * fnorm / pow(3.0, 1.5) / Jpt.Det() - 1.0;
+}
+
+double TMOP_Metric_304::EvalW(const DenseMatrix &Jpt) const
+{
+   // mu_304 = (I1b/3)^3/2 - 1.
+   ie.SetJacobian(Jpt.GetData());
+   return pow(ie.Get_I1b()/3.0, 1.5) - 1.0;
+}
+
+void TMOP_Metric_304::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   // mu_304 = (I1b/3)^3/2 - 1.
+   // P      = 3/2 * (I1b/3)^1/2 * dI1b / 3 = 1/2 * (I1b/3)^1/2 * dI1b.
+   ie.SetJacobian(Jpt.GetData());
+   P.Set(0.5 * sqrt(ie.Get_I1b()/3.0), ie.Get_dI1b());
+}
+
+void TMOP_Metric_304::AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                                const double weight, DenseMatrix &A) const
+{
+   // P  = 1/2 * (I1b/3)^1/2 * dI1b.
+   // dP = 1/12 * (I1b/3)^(-1/2) * (dI1b x dI1b) + 1/2 * (I1b/3)^1/2 * ddI1b.
+   ie.SetJacobian(Jpt.GetData());
+   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
+   ie.Assemble_TProd(weight / 12.0 / sqrt(ie.Get_I1b()/3.0),
+                     ie.Get_dI1b(), A.GetData());
+   ie.Assemble_ddI1b(weight / 2.0 * sqrt(ie.Get_I1b()/3.0), A.GetData());
+}
+
 double TMOP_Metric_311::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_311 = (det(J) - 1)^2 - det(J) + (det(J)^2 + eps)^{1/2}
@@ -868,6 +959,12 @@ void TMOP_Metric_315::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_ddI3b(2*weight*(ie.Get_I3b() - 1.0), A.GetData());
 }
 
+double TMOP_Metric_316::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_316 = 0.5 (det(J) + 1/det(J)) - 1.
+   return 0.5 * (Jpt.Det() + 1.0 / Jpt.Det()) - 1.0;
+}
+
 double TMOP_Metric_316::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_316 = mu_16_3D = 0.5*(I3b + 1/I3b) - 1
@@ -898,6 +995,16 @@ void TMOP_Metric_316::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_ddI3b(weight*(0.5 - 0.5/ie.Get_I3()), A.GetData());
 }
 
+double TMOP_Metric_321::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_321 = |J - J^-t|^2.
+   ie.SetJacobian(Jpt.GetData());
+   DenseMatrix invt(3);
+   CalcInverseTranspose(Jpt, invt);
+   invt.Add(-1.0, Jpt);
+   return invt.FNorm2();
+}
+
 double TMOP_Metric_321::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_321 = mu_21_3D = |J - J^{-t}|^2
@@ -946,6 +1053,119 @@ void TMOP_Metric_321::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_TProd(-3*c0*c3, ie.Get_dI3b(), A.GetData());
 }
 
+double TMOP_Metric_322::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_322 = 1 / (6 det(J)) |J - adj(J)^t|^2
+   DenseMatrix adj_J_t(3);
+   CalcAdjugateTranspose(Jpt, adj_J_t);
+   adj_J_t *= -1.0;
+   adj_J_t.Add(1.0, Jpt);
+   return 1.0 / 6.0 / Jpt.Det() * adj_J_t.FNorm2();
+}
+
+double TMOP_Metric_322::EvalW(const DenseMatrix &Jpt) const
+{
+   // mu_322 = 1 / (6 det(J)) |J - adj(J)^t|^2
+   //        = 1 / (6 det(J)) |J|^2 + 1/6 det(J) |J^{-1}|^2 - 1
+   //        = I1b / (I3b^-1/3) / 6 + I2b (I3b^1/3) / 6 - 1
+   ie.SetJacobian(Jpt.GetData());
+
+   return ie.Get_I1b() / pow(ie.Get_I3b(), 1.0/3.0) / 6.0 +
+          ie.Get_I2b() * pow(ie.Get_I3b(), 1.0/3.0) / 6.0 - 1.0;
+}
+
+void TMOP_Metric_322::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   // mu_322 = I1b (I3b^-1/3) / 6 + I2b (I3b^1/3) / 6 - 1
+   // P      =   1/6 (I3b^-1/3) dI1b - 1/18 I1b (I3b^-4/3) dI3b
+   //          + 1/6 (I3b^1/3) dI2b  + 1/18 I2b (I3b^-2/3) dI3b
+   ie.SetJacobian(Jpt.GetData());
+   P.Set(1.0/6.0 * pow(ie.Get_I3b(), -1.0/3.0),
+         ie.Get_dI1b());
+   P.Add(-1.0/18.0 * ie.Get_I1b() * pow(ie.Get_I3b(), -4.0/3.0),
+         ie.Get_dI3b());
+   P.Add(1.0/6.0 * pow(ie.Get_I3b(), 1.0/3.0),
+         ie.Get_dI2b());
+   P.Add(1.0/18.0 * ie.Get_I2b() * pow(ie.Get_I3b(), -2.0/3.0),
+         ie.Get_dI3b());
+}
+
+void TMOP_Metric_322::AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                                const double weight, DenseMatrix &A) const
+{
+   //  P =   1/6 (I3b^-1/3) dI1b - 1/18 I1b (I3b^-4/3) dI3b
+   //      + 1/6 (I3b^1/3) dI2b  + 1/18 I2b (I3b^-2/3) dI3b
+   // dP =   1/6 (I3b^-1/3) ddI1b - 1/18 (I3b^-4/3) (dI1b x dI3b)
+   //      - 1/18 I1b (I3b^-4/3) ddI3b
+   //      - 1/18 (I3b^-4/3) (dI3b x dI1b)
+   //      + 2/27 I1b (I3b^-7/3) (dI3b x dI3b)
+   //      + 1/6 (I3b^1/3) ddI2b + 1/18 (I3b^-2/3) (dI2b x dI3b)
+   //      + 1/18 I2b (I3b^-2/3) ddI3b
+   //      + 1/18 (I3b^-2/3) (dI3b x dI2b)
+   //      - 1/27 I2b (I3b^-5/3) (dI3b x dI3b)
+   ie.SetJacobian(Jpt.GetData());
+   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
+   const double p13 = weight * pow(ie.Get_I3b(),  1.0/3.0),
+                m13 = weight * pow(ie.Get_I3b(), -1.0/3.0),
+                m23 = weight * pow(ie.Get_I3b(), -2.0/3.0),
+                m43 = weight * pow(ie.Get_I3b(), -4.0/3.0),
+                m53 = weight * pow(ie.Get_I3b(), -5.0/3.0),
+                m73 = weight * pow(ie.Get_I3b(), -7.0/3.0);
+   ie.Assemble_ddI1b(1.0/6.0 * m13, A.GetData());
+   // Combines - 1/18 (I3b^-4/3) (dI1b x dI3b) - 1/18 (I3b^-4/3) (dI3b x dI1b).
+   ie.Assemble_TProd(-1.0/18.0 * m43,
+                     ie.Get_dI1b(), ie.Get_dI3b(), A.GetData());
+   ie.Assemble_ddI3b(-1.0/18.0 * ie.Get_I1b() * m43, A.GetData());
+   ie.Assemble_TProd(2.0/27.0 * ie.Get_I1b() * m73,
+                     ie.Get_dI3b(), A.GetData());
+   ie.Assemble_ddI2b(1.0/6.0 * p13, A.GetData());
+   // Combines + 1/18 (I3b^-2/3) (dI2b x dI3b) + 1/18 (I3b^-2/3) (dI3b x dI2b).
+   ie.Assemble_TProd(1.0/18.0 * m23,
+                     ie.Get_dI2b(), ie.Get_dI3b(), A.GetData());
+   ie.Assemble_ddI3b(1.0/18.0 * ie.Get_I2b() * m23, A.GetData());
+   ie.Assemble_TProd(-1.0/27.0 * ie.Get_I2b() * m53,
+                     ie.Get_dI3b(), A.GetData());
+}
+
+double TMOP_Metric_323::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_323 = |J|^3 - 3 sqrt(3) ln(det(J)) - 3 sqrt(3).
+   double fnorm = Jpt.FNorm();
+   return fnorm * fnorm * fnorm - 3.0 * sqrt(3.0) * (log(Jpt.Det()) + 1.0);
+}
+
+double TMOP_Metric_323::EvalW(const DenseMatrix &Jpt) const
+{
+   // mu_323 = I1^3/2 - 3 sqrt(3) ln(I3b) - 3 sqrt(3).
+   ie.SetJacobian(Jpt.GetData());
+   return pow(ie.Get_I1(), 1.5) - 3.0 * sqrt(3.0) * (log(ie.Get_I3b()) + 1.0);
+}
+
+void TMOP_Metric_323::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   // mu_323 = I1^3/2 - 3 sqrt(3) ln(I3b) - 3 sqrt(3).
+   // P      = 3/2 (I1^1/2) dI1 - 3 sqrt(3) (I3b^-1) dI3b.
+   ie.SetJacobian(Jpt.GetData());
+   P.Set(1.5 * sqrt(ie.Get_I1()), ie.Get_dI1());
+   P.Add(- 3.0 * sqrt(3.0) / ie.Get_I3b(), ie.Get_dI3b());
+}
+
+void TMOP_Metric_323::AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                                const double weight, DenseMatrix &A) const
+{
+   // P  =   3/2 (I1^1/2) dI1 - 3 sqrt(3) (I3b^-1) dI3b
+   // dP =   3/2 (I1^1/2) ddI1 + 3/4 (I1^-1/2) (dI1 x dI1)
+   //      - 3 sqrt(3) (I3b^-1) ddI3b + 3 sqrt(3) (I3b^-2) (dI3b x dI3b)
+   ie.SetJacobian(Jpt.GetData());
+   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
+   ie.Assemble_ddI1(weight * 1.5 * sqrt(ie.Get_I1()), A.GetData());
+   ie.Assemble_TProd(weight * 0.75 / sqrt(ie.Get_I1()),
+                     ie.Get_dI1(), A.GetData());
+   ie.Assemble_ddI3b(- weight * 3.0 * sqrt(3.0) / ie.Get_I3b(), A.GetData());
+   ie.Assemble_TProd(weight * 3.0 * sqrt(3.0) / ie.Get_I3b() / ie.Get_I3b(),
+                     ie.Get_dI3b(), A.GetData());
+}
+
 double TMOP_Metric_352::EvalW(const DenseMatrix &Jpt) const
 {
    // mu_352 = 0.5*(det(J) - 1)^2 / (det(J) - tau0)
@@ -989,6 +1209,45 @@ void TMOP_Metric_352::AssembleH(const DenseMatrix &Jpt,
    ie.Assemble_ddI3b(weight*(c - 0.5*c*c), A.GetData());
 }
 
+
+
+
+double TMOP_Metric_360::EvalWMatrixForm(const DenseMatrix &Jpt) const
+{
+   // mu_360 = |J|^3 / 3^(3/2) - det(J)
+   const double fnorm = Jpt.FNorm();
+   return fnorm * fnorm * fnorm / pow(3.0, 1.5) - Jpt.Det();
+}
+
+double TMOP_Metric_360::EvalW(const DenseMatrix &Jpt) const
+{
+   // mu_360 = (I1/3)^(3/2) - I3b.
+   ie.SetJacobian(Jpt.GetData());
+   return pow(ie.Get_I1()/3.0, 1.5) - ie.Get_I3b();
+}
+
+void TMOP_Metric_360::EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+{
+   // mu_360 = (I1/3)^(3/2) - I3b.
+   // P      = 3/2 * (I1/3)^1/2 * dI1 / 3 - dI3b
+   //        = 1/2 * (I1/3)^1/2 * dI1 - dI3b.
+   ie.SetJacobian(Jpt.GetData());
+   Add(0.5 * sqrt(ie.Get_I1()/3.0), ie.Get_dI1(), -1.0, ie.Get_dI3b(), P);
+}
+
+void TMOP_Metric_360::AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                                const double weight, DenseMatrix &A) const
+{
+   // P  = 1/2 * (I1/3)^1/2 * dI1 - dI3b.
+   // dP = 1/12 * (I1/3)^(-1/2) * (dI1 x dI1) + 1/2 * (I1/3)^1/2 * ddI1 - ddI3b
+   ie.SetJacobian(Jpt.GetData());
+   ie.SetDerivativeMatrix(DS.Height(), DS.GetData());
+   ie.Assemble_TProd(weight / 12.0 / sqrt(ie.Get_I1()/3.0),
+                     ie.Get_dI1(), A.GetData());
+   ie.Assemble_ddI1(weight / 2.0 * sqrt(ie.Get_I1()/3.0), A.GetData());
+   ie.Assemble_ddI3b(-weight, A.GetData());
+}
+
 double TMOP_AMetric_011::EvalW(const DenseMatrix &Jpt) const
 {
    MFEM_VERIFY(Jtr != NULL,
@@ -2917,7 +3176,6 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
       }
    }
 
-
    Vector d_detW_dx(dim);
    Vector d_Winv_dx(dim);
 
@@ -3740,7 +3998,6 @@ double TMOP_Integrator::ComputeMinDetT(const Vector &x,
       const int dof = fe->GetDof(), nsp = ir.GetNPoints();
 
       DSh.SetSize(dof, dim);
-      PMatI.SetSize(dof, dim);
       Vector posV(dof * dim);
       PMatI.UseExternalData(posV.GetData(), dof, dim);
 
@@ -3795,7 +4052,6 @@ double TMOP_Integrator::ComputeUntanglerMaxMuBarrier(const Vector &x,
       Jpt.SetSize(dim);
 
       DSh.SetSize(dof, dim);
-      PMatI.SetSize(dof, dim);
       Vector posV(dof * dim);
       PMatI.UseExternalData(posV.GetData(), dof, dim);
 

fem/tmop.hpp

@@ -26,8 +26,8 @@ protected:
    const DenseMatrix *Jtr; /**< Jacobian of the reference-element to
                                 target-element transformation. */
 
-   /** @brief The method SetTransformation() is hidden for TMOP_QualityMetric%s,
-       because it is not used. */
+   /** @brief The method HyperelasticModel::SetTransformation() is hidden
+       for TMOP_QualityMetric%s, because it is not used. */
    void SetTransformation(ElementTransformation &) { }
 
 public:
@@ -42,7 +42,13 @@ public:
        Jpt. */
    virtual void SetTargetJacobian(const DenseMatrix &Jtr_) { Jtr = &Jtr_; }
 
-   /** @brief Evaluate the strain energy density function, W = W(Jpt).
+   /** @brief Evaluates the metric in matrix form (opposed to invariant form).
+       Used for validating the invariant evaluations. */
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const
+   { return -1.0; /* not implemented -> checks would fail. */ }
+
+   /** @brief Evaluate the strain energy density function, W = W(Jpt), by using
+       the 2D or 3D matrix invariants, see linalg/invariants.hpp.
        @param[in] Jpt  Represents the target->physical transformation
                        Jacobian matrix. */
    virtual double EvalW(const DenseMatrix &Jpt) const = 0;
@@ -64,13 +70,11 @@ public:
 
        Computes weight * d(dW_dxi)_d(xj) at the current point, for all i and j,
        where x1 ... xn are the FE dofs. This function is usually defined using
-       the matrix invariants and their derivatives.
-   */
+       the matrix invariants and their derivatives. */
    virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
                           const double weight, DenseMatrix &A) const = 0;
 
-   /** @brief Return the metric ID.
-    */
+   /** @brief Return the metric ID. */
    virtual int Id() const { return 0; }
 };
 
@@ -96,6 +100,8 @@ public:
       }
    }
 
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -272,7 +278,10 @@ protected:
    mutable InvariantsEvaluator2D<double> ie;
 
 public:
-   // W = 0.5|J|^2 / det(J) - 1.
+   // W = 0.5 |J|^2 / det(J) - 1.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = 0.5 I1b - 1.
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -428,7 +437,9 @@ protected:
 
 public:
    // W = |J^t J|^2 / det(J)^2 - 2|J|^2 / det(J) + 2
-   //   = I1b (I1b - 2).
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1b (I1b - 2).
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -570,14 +581,17 @@ public:
                           const double weight, DenseMatrix &A) const;
 };
 
-/// 3D barrier Shape (S) metric.
+/// 3D barrier Shape (S) metric, well-posed (polyconvex & invex).
 class TMOP_Metric_301 : public TMOP_QualityMetric
 {
 protected:
    mutable InvariantsEvaluator3D<double> ie;
 
 public:
-   // W = |J| |J^-1| / 3 - 1.
+   // W = 1/3 |J| |J^-1| - 1.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = 1/3 sqrt(I1b * I2b) - 1
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -586,14 +600,17 @@ public:
                           const double weight, DenseMatrix &A) const;
 };
 
-/// 3D barrier Shape (S) metric.
+/// 3D barrier Shape (S) metric, well-posed (polyconvex & invex).
 class TMOP_Metric_302 : public TMOP_QualityMetric
 {
 protected:
    mutable InvariantsEvaluator3D<double> ie;
 
 public:
-   // W = |J|^2 |J^-1|^2 / 9 - 1.
+   // W = |J|^2 |J^{-1}|^2 / 9 - 1.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1b * I2b / 9 - 1.
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -604,14 +621,17 @@ public:
    virtual int Id() const { return 302; }
 };
 
-/// 3D barrier Shape (S) metric.
+/// 3D barrier Shape (S) metric, well-posed (polyconvex & invex).
 class TMOP_Metric_303 : public TMOP_QualityMetric
 {
 protected:
    mutable InvariantsEvaluator3D<double> ie;
 
 public:
-   // W = |J|^2 / (3 * det(J)^(2/3)) - 1.
+   // W = |J|^2 / 3 / det(J)^(2/3) - 1.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1b / 3 - 1.
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -622,6 +642,27 @@ public:
    virtual int Id() const { return 303; }
 };
 
+/// 3D barrier Shape (S) metric, well-posed (polyconvex & invex).
+class TMOP_Metric_304 : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // W = |J|^3 / 3^(3/2) / det(J) - 1.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = (I1b/3)^3/2 - 1.
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 304; }
+};
+
 /// 3D Size (V) untangling metric.
 class TMOP_Metric_311 : public TMOP_QualityMetric
 {
@@ -662,7 +703,7 @@ public:
    virtual int Id() const { return 313; }
 };
 
-/// 3D non-barrier Size (V) metric.
+/// 3D non-barrier metric without a type.
 class TMOP_Metric_315 : public TMOP_QualityMetric
 {
 protected:
@@ -680,16 +721,17 @@ public:
    virtual int Id() const { return 315; }
 };
 
-/// 3D barrier Size (V) metric.
+/// 3D barrier metric without a type.
 class TMOP_Metric_316 : public TMOP_QualityMetric
 {
 protected:
    mutable InvariantsEvaluator3D<double> ie;
 
 public:
-   // W = 0.5( sqrt(det(J)) - 1 / sqrt(det(J)) )^2
-   //   = 0.5( det(J) - 1 )^2 / det(J)
-   //   = 0.5( det(J) + 1/det(J) ) - 1.
+   // W = 0.5 (det(J) + 1/det(J)) - 1.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = 0.5 (I3b + 1/I3b) - 1.
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -698,7 +740,7 @@ public:
                           const double weight, DenseMatrix &A) const;
 };
 
-/// 3D barrier Shape+Size (VS) metric.
+/// 3D barrier Shape+Size (VS) metric, well-posed (invex).
 class TMOP_Metric_321 : public TMOP_QualityMetric
 {
 protected:
@@ -706,6 +748,9 @@ protected:
 
 public:
    // W = |J - J^-t|^2.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1 + I2/I3 - 6.
    virtual double EvalW(const DenseMatrix &Jpt) const;
 
    virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
@@ -716,6 +761,48 @@ public:
    virtual int Id() const { return 321; }
 };
 
+/// 3D barrier Shape+Size (VS) metric, well-posed (invex).
+class TMOP_Metric_322 : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // W = |J - adjJ^-t|^2.
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1b / (I3b^-1/3) / 6 + I2b (I3b^1/3) / 6 - 1
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 322; }
+};
+
+/// 3D barrier Shape+Size (VS) metric, well-posed (invex).
+class TMOP_Metric_323 : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // W = |J|^3 - 3 sqrt(3) ln(det(J)) - 3 sqrt(3).
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = I1^3/2 - 3 sqrt(3) ln(I3b) - 3 sqrt(3).
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 323; }
+};
+
 /// 3D barrier Shape+Size (VS) metric (polyconvex).
 class TMOP_Metric_328 : public TMOP_Combo_QualityMetric
 {
@@ -760,7 +847,7 @@ public:
    virtual ~TMOP_Metric_332() { delete sh_metric; delete sz_metric; }
 };
 
-/// 3D barrier Shape+Size (VS) metric (polyconvex).
+/// 3D barrier Shape+Size (VS) metric, well-posed (polyconvex).
 class TMOP_Metric_333 : public TMOP_Combo_QualityMetric
 {
 protected:
@@ -781,7 +868,7 @@ public:
    virtual ~TMOP_Metric_333() { delete sh_metric; delete sz_metric; }
 };
 
-/// 3D barrier Shape+Size (VS) metric (polyconvex).
+/// 3D barrier Shape+Size (VS) metric, well-posed (polyconvex).
 class TMOP_Metric_334 : public TMOP_Combo_QualityMetric
 {
 protected:
@@ -799,10 +886,37 @@ public:
       AddQualityMetric(sz_metric, gamma_);
    }
 
+   virtual int Id() const { return 334; }
+   double GetGamma() const { return gamma; }
+
    virtual ~TMOP_Metric_334() { delete sh_metric; delete sz_metric; }
 };
 
-/// Shifted barrier form of 3D metric 16 (volume, ideal barrier metric), 3D
+/// 3D barrier Shape+Size (VS) metric, well-posed (polyconvex).
+class TMOP_Metric_347 : public TMOP_Combo_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator2D<double> ie;
+   double gamma;
+   TMOP_QualityMetric *sh_metric, *sz_metric;
+
+public:
+   TMOP_Metric_347(double gamma_) : gamma(gamma_),
+      sh_metric(new TMOP_Metric_304),
+      sz_metric(new TMOP_Metric_316)
+   {
+      // (1-gamma) mu_304 + gamma mu_316
+      AddQualityMetric(sh_metric, 1.-gamma_);
+      AddQualityMetric(sz_metric, gamma_);
+   }
+
+   virtual int Id() const { return 347; }
+   double GetGamma() const { return gamma; }
+
+   virtual ~TMOP_Metric_347() { delete sh_metric; delete sz_metric; }
+};
+
+/// 3D shifted barrier form of metric 316 (not typed).
 class TMOP_Metric_352 : public TMOP_QualityMetric
 {
 protected:
@@ -821,6 +935,27 @@ public:
                           const double weight, DenseMatrix &A) const;
 };
 
+/// 3D non-barrier Shape (S) metric.
+class TMOP_Metric_360 : public TMOP_QualityMetric
+{
+protected:
+   mutable InvariantsEvaluator3D<double> ie;
+
+public:
+   // W = |J|^3 / 3^(3/2) - det(J).
+   virtual double EvalWMatrixForm(const DenseMatrix &Jpt) const;
+
+   // W = (I1b/3)^3/2 - 1.
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const;
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const;
+
+   virtual int Id() const { return 360; }
+};
+
 /// A-metrics
 /// 2D barrier Shape (S) metric (polyconvex).
 class TMOP_AMetric_011 : public TMOP_QualityMetric

fem/tmop/tmop_pa.hpp

@@ -75,7 +75,7 @@ public:
 //  - the second argument (kernel) is the name of the kernel
 //  - the arguments of the kernel (...) captured as __VA_ARGS__
 //
-// This call will output the followings:
+// This call will output the following:
 //  1. forward declaration of the kernel
 //  2. kernel pointer declaration
 //  3. struct K##name##_T definition which holds the keys/kernels map

general/communication.cpp

@@ -340,7 +340,7 @@ const MPI_Datatype MPITypeMap<int>::mpi_type = MPI_INT;
 const MPI_Datatype MPITypeMap<double>::mpi_type = MPI_DOUBLE;
 
 
-GroupCommunicator::GroupCommunicator(GroupTopology &gt, Mode m)
+GroupCommunicator::GroupCommunicator(const GroupTopology &gt, Mode m)
    : gtopo(gt), mode(m)
 {
    group_buf_size = 0;

general/communication.hpp

@@ -210,7 +210,7 @@ public:
    };
 
 protected:
-   GroupTopology &gtopo;
+   const GroupTopology &gtopo;
    Mode mode;
    Table group_ldof;
    Table group_ltdof; // only for groups for which this processor is master.
@@ -233,7 +233,7 @@ public:
        - initialize the Table reference returned by GroupLDofTable() and then
          call Finalize().
    */
-   GroupCommunicator(GroupTopology &gt, Mode m = byNeighbor);
+   GroupCommunicator(const GroupTopology &gt, Mode m = byNeighbor);
 
    /** @brief Initialize the communicator from a local-dof to group map.
        Finalize() is called internally. */
@@ -255,7 +255,7 @@ public:
    void SetLTDofTable(const Array<int> &ldof_ltdof);
 
    /// Get a reference to the associated GroupTopology object
-   GroupTopology &GetGroupTopology() { return gtopo; }
+   const GroupTopology &GetGroupTopology() { return gtopo; }
 
    /// Get a const reference to the associated GroupTopology object
    const GroupTopology &GetGroupTopology() const { return gtopo; }

general/forall.hpp

@@ -36,7 +36,7 @@ const int MAX_Q1D = 14;
 #define MFEM_PRAGMA(X) _Pragma(#X)
 
 // MFEM_UNROLL pragma macro that can be used inside MFEM_FORALL macros.
-#if defined(MFEM_USE_CUDA)
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__)
 #define MFEM_UNROLL(N) MFEM_PRAGMA(unroll(N))
 #else
 #define MFEM_UNROLL(N)

general/gecko.hpp

@@ -277,7 +277,7 @@ public:
    // compute weighted mean from a weighted sum
    virtual Float mean(const WeightedSum& sum) const = 0;
 
-   // compute k'th iteration bond for egde of length l and weight w
+   // compute k'th iteration bond for edge of length l and weight w
    virtual Float bond(Float w, Float l, uint k) const = 0;
 
    // compute position that minimizes weighted distance to a point set

general/mem_manager.hpp

@@ -1141,7 +1141,7 @@ inline void Memory<T>::SyncAlias(const Memory &base, int alias_size) const
 template <typename T>
 inline MemoryType Memory<T>::GetMemoryType() const
 {
-   if (!(flags & VALID_DEVICE)) { return h_mt; }
+   if (h_ptr == nullptr || !(flags & VALID_DEVICE)) { return h_mt; }
    return MemoryManager::GetDeviceMemoryType_(h_ptr, flags & ALIAS);
 }