Improve #include logic.

Update the 'raja-cuda' and 'occa-cuda' backends to use the
default stream 0. Optimize OperatorJacobiSmoother::Mult by fusing kernels as much as possible.
2020-10-29 05:20:58 -07:00 · 2020-10-29 04:28:15 -07:00 · 2020-10-24 19:07:23 -07:00 · 2020-10-24 11:27:42 -04:00 · 2020-10-23 07:19:11 -07:00 · 2020-10-22 13:51:26 -07:00
128 changed files with 7468 additions and 2552 deletions
@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 30
+
+# Number of days of inactivity before an Issue or Pull Request with the stale
+# label is closed.  Set to false to disable. If disabled, issues still need to
+# be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale.
+# Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set
+# to `[]` to disable
+exemptLabels:
+  - bug
+  - WIP
+  - ready-for-review
+  - in-review
+  - in-next
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking an issue as stale
+staleLabel: stale
+
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  :warning: This issue or PR has been automatically marked as stale because it has not
+  had any activity in the last month. *If no activity occurs in the next week, it will 
+  be automatically closed.* Thank you for your contributions.
+
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed
@@ -102,28 +102,29 @@ examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*

-examples/sundials/ex9
-examples/sundials/ex1[06]
-examples/sundials/ex9p
-examples/sundials/ex1[06]p
+examples/amgx/ex1
+examples/amgx/ex1p
+examples/amgx/.logamgx
+examples/amgx/refined.mesh
+examples/amgx/sol.gf
+examples/amgx/mesh.*
+examples/amgx/sol.*

-examples/sundials/ex9.mesh
-examples/sundials/ex9-mesh.*
-examples/sundials/ex9-init.*
-examples/sundials/ex9-final.*
-examples/sundials/Example9*
-examples/sundials/deformed.*
-examples/sundials/velocity.*
-examples/sundials/elastic_energy.*
-examples/sundials/ex16.mesh
-examples/sundials/ex16-mesh.*
-examples/sundials/ex16-init.*
-examples/sundials/ex16-final.*
-examples/sundials/Example16*
+examples/gingko/ex1
+examples/gingko/refined.mesh
+examples/gingko/sol.gf
+examples/gingko/mesh.*
+examples/gingko/sol.*
+
+examples/hiop/ex9
+examples/hiop/ex9p
+examples/hiop/ex9.mesh
+examples/hiop/ex9-mesh.*
+examples/hiop/ex9-init.*
+examples/hiop/ex9-final.*

 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
-
 examples/petsc/mesh.*
 examples/petsc/sol.*
 examples/petsc/sol_p.*
@@ -141,28 +142,51 @@ examples/petsc/mode_*

 examples/pumi/ex1
 examples/pumi/ex[126]p
-
-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
-
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
 examples/pumi/sol.*
 examples/pumi/displaced.mesh

+examples/sundials/ex9
+examples/sundials/ex1[06]
+examples/sundials/ex9p
+examples/sundials/ex1[06]p
+examples/sundials/ex9.mesh
+examples/sundials/ex9-mesh.*
+examples/sundials/ex9-init.*
+examples/sundials/ex9-final.*
+examples/sundials/Example9*
+examples/sundials/deformed.*
+examples/sundials/velocity.*
+examples/sundials/elastic_energy.*
+examples/sundials/ex16.mesh
+examples/sundials/ex16-mesh.*
+examples/sundials/ex16-init.*
+examples/sundials/ex16-final.*
+examples/sundials/Example16*
+
+examples/superlu/ex1p
+examples/superlu/mesh.*
+examples/superlu/sol.*
+
+miniapps/adjoint/cvsRoberts_ASAi_dns
+miniapps/adjoint/adjoint_advection_diffusion
+
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
-
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*

+miniapps/gslib/field-diff
+miniapps/gslib/field-interp
+miniapps/gslib/findpts
+miniapps/gslib/pfindpts
+
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid
@@ -176,7 +200,6 @@ miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
-
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -190,9 +213,25 @@ miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh

+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output
+
+miniapps/nurbs/nurbs_ex1
+miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/refined.mesh
+miniapps/nurbs/mesh.*
+miniapps/nurbs/sol.*
+miniapps/nurbs/mode_*
+miniapps/nurbs/Example1*
+
 miniapps/performance/ex1
 miniapps/performance/ex1p
-
 miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
@@ -210,7 +249,6 @@ miniapps/toys/rubik
 miniapps/toys/snake
 miniapps/toys/lissajous
 miniapps/toys/mondrian
-
 miniapps/toys/snake-init.mesh
 miniapps/toys/snake-user.mesh
 miniapps/toys/snake-joined.mesh
@@ -225,37 +263,13 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh

-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/refined.mesh
-miniapps/nurbs/mesh.*
-miniapps/nurbs/sol.*
-miniapps/nurbs/mode_*
-miniapps/nurbs/Example1*
-
-miniapps/gslib/field-diff
-miniapps/gslib/field-interp
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
+tests/unit/ceed_tests

 # Test script output
 tests/scripts/*.err
@@ -37,7 +37,7 @@ build_mfem_debug_ser_lassen:
    - mkdir -p ${BUILD_PATH}
    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CUDA_ARCH=sm_70
+    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70

 # Sanity check
 sanitycheck_mfem_ser_lassen:
@@ -11,6 +11,16 @@
 Version 4.1.1 (development)
 ===========================

+libCEED integration improvements
+--------------------------------
+- Add support for all types of (scalar) Coefficient.
+
+- Add support for VectorMassIntegrator and VectorDiffusionIntegrator.
+
+- Add support for AssemblyLevel::NONE for MassIntegrator, DiffusionIntegrator,
+  VectorMassIntegrator, and VectorDiffusionIntegrator. This level of assembly
+  fully applies on the fly the finite element operator.
+
 Meshing improvements
 --------------------
 - The graph linear ordering library Gecko, previously an external dependency, is
@@ -56,10 +66,30 @@ Performance improvements

 Improved GPU capabilities
 -------------------------
+- Added a new solver class for simple integration with NVIDIA's multigrid
+  library, AmgX. The AmgX class is designed to work as a standalone solver or
+  preconditioner for existing MFEM solvers. It uses MFEM's sparse matrix format
+  for serial runs and the HypreParMatrix format for parallel runs.
+
+  The new solver may be configured to run with one GPU per MPI rank or with more
+  MPI ranks than GPUs. In the latter case, matrices and vectors are consolidated
+  to ranks communicating with the GPUs and the solution is then broadcasted.
+
+  Although CUDA is required to build, the AmgX support is compatible with the
+  MFEM CPU device configuration. The examples/amgx folder illustrates how to
+  integrate AmgX in existing MFEM applications.
+
+  The AmgX solver class is partially based on: "AmgXWrapper: An interface
+  between PETSc and the NVIDIA AmgX library", by Pi-Yueh Chuang and Lorena
+  A. Barba, doi:10.21105/joss.00280.
+
 - Added support for Chebyshev accelerated polynomial smoother on GPU.

 - Optimized AMD/HIP kernel support.

+- Enabled HIP support in the libCEED integration, which is now available via the
+  "ceed-hip" device backend.
+
 - Added a Full Assembly mode compatible with Device kernel execution. This
  assembly level builds on top of the current Element Assembly kernels to
  compute a global sparse matrix. All integrators supported by element assembly
@@ -67,14 +97,15 @@ Improved GPU capabilities

 - Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.

- Added AmgX solver class for simple integration with NVIDIA's multigrid library.
-
 - Added support for BlockOperator on GPU. See the updated Example 5.

 - Added partial assembly and GPU support for complex operators, including the
  classes ComplexOperator, [Par]ComplexGridFunction, [Par]ComplexLinearForm, and
  [Par]SesquilinearForm. See the updated Example 22.

+- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
+  modification of Example 9/9p.
+
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -101,6 +132,9 @@ Discretization improvements
  Additionally, new LinearForm integrators were also added which make use of
  these new QuadratureFunction coefficient classes.

+- Non-conforming meshes are now supported with block nonlinear forms. See the
+  updated Example 19/19p.
+
 - Added support face integrals on the boundaries of NURBS meshes.

 - Added support for interpolation of functions in L2, H(div) and H(curl)
@@ -111,6 +145,12 @@ Discretization improvements
  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
  the tests/convergence directory for sample usage.

+- The C-function based coefficient classes (FunctionCoefficient,
+  VectorFunctionCoefficient, and MatrixFunctionCoefficient) now use the more
+  general std::function class template. This allows the classes to be backward
+  compatible (i.e. they can still work with C-functions) and, in addition,
+  support any "callable", e.g. lambda functions.
+
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -119,6 +159,9 @@ Linear and nonlinear solvers
 - Added initial support for h- and p-multigrid solvers and preconditioners for
  matrix-based and matrix-free discretizations with basic GPU capability.

+- Added wrappers for Hypre's flexible GMRES solver and the new parallel ILU
+  preconditioner. The latter requires hypre version 2.19.0 or later.
+
 - Added a new IterativeSolverMonitor class that allows to monitor the residual
  and solution during the solving process of an IterativeSolver after every
  iteration.
@@ -139,6 +182,15 @@ Linear and nonlinear solvers
 - Added partially assembled convergent diagonal preconditioner for adaptively
  refined meshes (i.e. non-conforming finite element spaces), see Example 6/6p.

+- Upgraded SuperLU interface to use SuperLU_DIST 6.3.1. Added a simple SuperLU
+  example in the new directory examples/superlu.
+
+- Extended the KINSOL (SUNDIALS) nonlinear solver interface to support the
+  Jacobian-free Newton-Krylov method. A usage example is shown in Example 10p.
+
+- Added an interface to the MKL CPardiso solver -- an MPI-parallel sparse direct
+  solver developed by Intel. See Example 11p for an illustration of its usage.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -213,6 +265,8 @@ Improved testing
  triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
  and the updated CONTRIBUTING.md file.

+- Add tests for the libCEED integration in MFEM.
+
 - Added testing of the parallel mesh format in tests/par-mesh-format.

 Miscellaneous
@@ -225,6 +279,9 @@ Miscellaneous
  ParaView v5.8.1. In addition, ADIOS2 allows for setting a user-defined number
  of data substreams/subfiles at scale. See examples 5, 9, 12, 16.

+- Added VTU output of boundary elements and attributes and parallel VTU (PVTU)
+  output of parallel meshes for visualization using ParaView.
+
 - The integration order used in the ComputeLpError and ComputeElementLpError
  methods of class GridFunction has been increased.

@@ -233,6 +290,9 @@ Miscellaneous
 - Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
  as DEBUG is sometimes used as a macro.

+- Change the IntegrationRule inside VectorDiffusionIntegrator to use the same
+  quadrature as DiffusionIntegrator.
+

 Version 4.1, released on March 10, 2020
 =======================================
@@ -240,12 +240,14 @@ endif()

 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
-  if (NOT MFEM_USE_MPI)
-    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
-  else()
-    find_package(SUNDIALS REQUIRED
-      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
+  set(SUNDIALS_COMPONENTS CVODES ARKODE KINSOL NVector_Serial)
+  if (MFEM_USE_MPI)
+    list(APPEND SUNDIALS_COMPONENTS NVector_Parallel NVector_MPIPlusX)
  endif()
+  if (MFEM_USE_CUDA)
+    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
+  endif()
+  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

 # Mesquite
@@ -296,8 +298,7 @@ if (MFEM_USE_CEED)
 endif()

 if (MFEM_USE_AMGX)
-  include_directories(${AMGX_DIR}/include)
-  link_directories(${AMGX_DIR}/lib64 -L${AMGX_DIR}/lib -Xlinker=-rpath=${CUDA_HOME}/lib64)
+  find_package(AMGX REQUIRED)
 endif()

 if (MFEM_USE_CONDUIT)
@@ -351,6 +352,12 @@ if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
 endif()

+if (MFEM_USE_MKL_CPARDISO)
+  if (MFEM_USE_MPI)
+   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
+  endif()
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -377,7 +384,7 @@ endif()
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE AMGX)
+    CUSPARSE MKL_CPARDISO AMGX)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -98,15 +98,18 @@ The MFEM source code has the following structure:
  │   └── web
  │       └── examples
  ├── examples
+  │   ├── amgx
  │   ├── ginkgo
  │   ├── hiop
  │   ├── petsc
  │   ├── pumi
  │   └── sundials
+  |   └── superlu
  ├── fem
  │   └── libceed
  ├── general
  ├── linalg
+  │   └── simd
  ├── mesh
  ├── miniapps
  │   ├── adjoint
@@ -114,11 +117,14 @@ The MFEM source code has the following structure:
  │   ├── electromagnetics
  │   ├── gslib
  │   ├── meshing
+  │   ├── navier
  │   ├── nurbs
  │   ├── performance
  │   ├── tools
  │   └── toys
  └── tests
+      ├── convergence
+      ├── par-mesh-format
      ├── scripts
      ├── unit
      │   ├── ...
@@ -344,24 +344,27 @@ MFEM_USE_SUPERLU = YES/NO
   SuperLURowLocMatrix a distributed CSR matrix class needed by SuperLU. When
   enabled, this option uses the SUPERLU_* library options, see below.

+MFEM_USE_SUPERLU5 = YES/NO
+   If SuperLU functionality is enabled, use the older 5.1.0 version rather than
+   the more recent 6+ versions.
+
 MFEM_USE_STRUMPACK = YES/NO
   Enable MFEM functionality based on the STRUMPACK sparse direct solver and
   preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
   classes. When enabled, this option uses the STRUMPACK_* library options, see
   below.

-MFEM_USE_AMGX = YES/NO
-   Enable MFEM functionality based on the AMGX multigrid library from 
-   NVIDIA. When enabled, this options enables the user to use SparseMatrices
-   and HypreParMatrices to solve linear systems using routines from the AMGX
-   library.
-
 MFEM_USE_GINKGO = YES/NO
   Enable MFEM functionality based on the Ginkgo library, which provides
   iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
   https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
   Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.

+MFEM_USE_AMGX = YES/NO
+   Enable MFEM functionality based on the AmgX multigrid library from NVIDIA.
+   Allows the user to use SparseMatrices and HypreParMatrices to solve linear
+   systems with the routines from the AmgX library.
+
 MFEM_USE_GNUTLS = YES/NO
   Enable secure socket support in class socketstream, using the auxiliary
   GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -492,6 +495,13 @@ MFEM_USE_CEED = YES/NO
   library for performant high-order operator evaluation developed by the Center
   for Efficient Exascale Discretizations in the Exascale Computing Project.

+MFEM_USE_MKL_CPARDISO = YES/NO
+   Enables the interface to MKL CPardiso: the Intel MKL Parallel Direct Sparse
+   Solver for Clusters. Make sure to set the correct values for MKL_MPI_WRAPPER
+   and MKL_LIBRARY_SUBDIR as shown in defaults.mk. If you configure MFEM with
+   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
+   OpenMP capabilities are disabled at link time.
+
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -541,11 +551,14 @@ The specific libraries and their options are:
  Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
  Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
  Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
+  When MFEM_USE_CUDA is enabled, only SUNDIALS v5.4.0+ is supported.
  If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
  both MPI and hypre.
+  If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
+  for CUDA.
  URL: http://computation.llnl.gov/projects/sundials/sundials-software
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0.
+  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA suppport.

 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
  URL: http://trilinos.org/oldsite/packages/mesquite
@@ -580,6 +593,11 @@ The specific libraries and their options are:
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT (Not used), GINKGO_LIB.

+- AmgX (optional), used when MFEM_USE_AMGX = YES.
+  URL: https://github.com/NVIDIA/AMGX
+  Options: AMGX_OPT, AMGX_LIB.
+  Versions: AmgX >= 2.1, older versions may work too.
+
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
  GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
  one can get the library through the Homebrew package manager (http://brew.sh).
@@ -651,16 +669,16 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.5.

+- MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
+  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
+  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
+  Versions: Intel MKL >= 2020.
+
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
  Versions: CUDA >= 9.1, older versions may work too.

- AMGX (optional), used when MFEM_USE_AMGX = YES.
-  URL: https://github.com/NVIDIA/AMGX
-  Options: AMGX_OPT, AMGX_LIB.
-  Versions: AMGX >= 2.1, older versions may work too.
-
 - HIP (optional), used when MFEM_USE_HIP = YES.
  URL: https://rocm.github.io/ROCmInstall.html
  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
@@ -668,13 +686,13 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.0.9.
+  Versions: OCCA >= 1.1.0.

 - libCEED (optional), used when MFEM_USE_CEED = YES.
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED > 0.6, git-hash bdfed75.
+  Versions: libCEED >= 0.7.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -817,8 +835,8 @@ MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
-MFEM_USE_AMGX
 MFEM_USE_GINKGO
+MFEM_USE_AMGX
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
@@ -867,11 +885,12 @@ The CMake build system adds auto-detection for the following packages/libraries:

 - HYPRE
 - METIS - The option MFEM_USE_METIS_5 is auto-detected.
+ - ParMETIS
 - MESQUITE
 - SuiteSparse
 - SuperLUDist, STRUMPACK
- - ParMETIS
 - Ginkgo
+ - AMGX
 - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
 - NETCDF
 - MPFR
@@ -25,7 +25,8 @@ This project distributes the sources of several external software products with
 their own respective licenses which can be found in their code and attached
 license files. These software products and their licenses are as follows:

+* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
+* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
-* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
 * Zstr (general/zstr.hpp) -- MIT license
@@ -34,6 +34,7 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
+set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
@@ -74,6 +74,9 @@
 // [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 #cmakedefine MFEM_USE_LEGACY_OPENMP

+// Internal MFEM option: enable group/batch allocation for some small objects.
+#cmakedefine MFEM_USE_MEMALLOC
+
 // Enable MFEM functionality based on the Mesquite library.
 #cmakedefine MFEM_USE_MESQUITE

@@ -86,12 +89,12 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

-// Internal MFEM option: enable group/batch allocation for some small objects.
-#cmakedefine MFEM_USE_MEMALLOC
-
 // Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO

+// Enable MFEM functionality based on the AmgX library
+#cmakedefine MFEM_USE_AMGX
+
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS

@@ -126,9 +129,6 @@
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA

-// Enable MFEM functionality based on the AMGX library
-#cmakedefine MFEM_USE_AMGX
-
 // Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA

@@ -159,4 +159,7 @@
 // library.
 #cmakedefine MFEM_USE_SIMMETRIX

+// Enable interface to the MKL CPardiso library.
+#cmakedefine MFEM_USE_MKL_CPARDISO
+
 #endif // MFEM_CONFIG_HEADER
@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - AMGX_FOUND
+#   - AMGX_LIBRARIES
+#   - AMGX_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
+mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
+  "Paths to headers required by AMGX." "Libraries required by AMGX.")
@@ -0,0 +1,32 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - MKL_CPARDISO_FOUND
+#   - MKL_CPARDISO_LIBRARIES
+#   - MKL_CPARDISO_INCLUDE_DIRS
+
+if(NOT MKL_MPI_WRAPPER_LIB)
+  message(FATAL_ERROR "MKL CPardiso enabled but no MKL MPI Wrapper lib specified")
+endif()
+
+if(NOT MKL_LIBRARY_DIR)
+  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
+  set(MKL_LIBRARY_DIR "lib")
+endif()
+
+include(MfemCmakeUtilities)
+mfem_find_package(MKL_CPARDISO MKL_CPARDISO
+    MKL_CPARDISO_DIR "include" mkl_cluster_sparse_solver.h ${MKL_LIBRARY_DIR} mkl_core
+  "Paths to headers required by MKL CPardiso." "Libraries required by MKL CPARDISO."
+  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
+  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential
+  ADD_COMPONENT MKL_MPI_WRAPPER "include" "" ${MKL_LIBRARY_DIR} ${MKL_MPI_WRAPPER_LIB})
@@ -20,10 +20,14 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  "Paths to headers required by SUNDIALS." "Libraries required by SUNDIALS."
  ADD_COMPONENT NVector_Serial
    "include" nvector/nvector_serial.h "lib" sundials_nvecserial
-  ADD_COMPONENT NVector_Parallel
-    "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
+  ADD_COMPONENT NVector_Cuda
+    "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
  ADD_COMPONENT NVector_ParHyp
    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
+  ADD_COMPONENT NVector_Parallel
+    "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
+  ADD_COMPONENT NVector_MPIPlusX
+    "include" nvector/nvector_mpiplusx.h "lib" sundials_nvecmpiplusx
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
@@ -738,10 +738,11 @@ function(mfem_export_mk_files)
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
-      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
+      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
+      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
+      MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -45,6 +45,9 @@
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
+#ifdef MFEM_USE_MKL_CPARDISO
+#error Building with MKL CPARDISO (MFEM_USE_MKL_CPARDISO=YES) requires MPI (MFEM_USE_MPI=YES)
+#endif
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -93,6 +93,7 @@

 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
+// #define MFEM_USE_SUPERLU5

 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
@@ -100,6 +101,9 @@
 // Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO

+// Enable MFEM functionality based on the AmgX library.
+// #define MFEM_USE_AMGX
+
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS

@@ -137,9 +141,6 @@
 // Requires a CUDA compiler (nvcc).
 // #define MFEM_USE_CUDA

-// Enable MFEM functionality based on the AMGX library.
-// #define MFEM_USE_AMGX
-
 // Build the AMD GPU/HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 // #define MFEM_USE_HIP
@@ -166,4 +167,7 @@
 // library.
 // #define MFEM_USE_SIMMETRIX

+// Enable interface to the MKL CPardiso library.
+// #define MFEM_USE_MKL_CPARDISO
+
 #endif // MFEM_CONFIG_HEADER
@@ -32,8 +32,10 @@ MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
+MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
+MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -45,7 +47,6 @@ MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
-MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
@@ -53,6 +54,7 @@ MFEM_USE_CEED          = @MFEM_USE_CEED@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
+MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -33,8 +33,10 @@ option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
+option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
+option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -52,6 +54,7 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
+option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -88,7 +91,7 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
+# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
    "Path to the SUNDIALS library.")
@@ -109,7 +112,7 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
    "Additional packages required by ParMETIS.")

-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_5.1.0" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
    "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
 set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
@@ -145,6 +148,8 @@ set(ScaLAPACK_TARGET_NAMES scalapack)

 set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")

+set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
+
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -180,9 +185,12 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
    "Packages that HiOp depends on.")

+set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
+set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
+set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
+
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AMGX")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")

@@ -120,8 +120,10 @@ MFEM_USE_SUNDIALS      = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
+MFEM_USE_SUPERLU5      = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
+MFEM_USE_AMGX          = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -133,7 +135,6 @@ MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
 MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
-MFEM_USE_AMGX          = NO
 MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
@@ -141,6 +142,7 @@ MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
+MFEM_USE_MKL_CPARDISO  = NO

 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -190,15 +192,19 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
+# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
+SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
+SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
 -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
-   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
+   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
+endif
+ifeq ($(MFEM_USE_CUDA),YES)
+   SUNDIALS_LIB += -lsundials_nveccuda
 endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
@@ -217,9 +223,15 @@ SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
 -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)

 # SuperLU library configuration
-SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
-SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
-SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
+ifeq ($(MFEM_USE_SUPERLU5),YES)
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
+   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0   
+else
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
+   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
+endif

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
 # STRUMPACK >= v2.2.0)
@@ -252,7 +264,13 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
+ -lginkgo_omp -lginkgo_cuda -lginkgo_reference
+
+# AmgX library configuration
+AMGX_DIR = @MFEM_DIR@/../amgx
+AMGX_OPT = -I$(AMGX_DIR)/include
+AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -373,10 +391,14 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

-# AMGX library configuration
-AMGX_DIR = @MFEM_DIR@/../amgx
-AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -L$(CUDA_HOME)/lib64 -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx -Xlinker=-rpath=$(CUDA_HOME)/lib64
+# MKL CPardiso library configuration
+MKL_CPARDISO_DIR ?=
+MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
+MKL_LIBRARY_SUBDIR ?= lib
+MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
+MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
+                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core

 # If YES, enable some informational messages
 VERBOSE = NO
@@ -127,6 +127,11 @@ if (MFEM_USE_GINKGO)
  add_subdirectory(ginkgo)
 endif()

+# Include the examples/amgx directory if AmgX is enabled
+if (MFEM_USE_AMGX)
+  add_subdirectory(amgx)
+endif()
+
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -140,4 +145,3 @@ endif()
 if (MFEM_USE_HIOP)
  add_subdirectory(hiop)
 endif()
-
@@ -0,0 +1,79 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(AMGX_EXAMPLES_SRCS)
+list(APPEND AMGX_EXAMPLES_SRCS
+  ex1.cpp
+  )
+
+if (MFEM_USE_MPI)
+  list(APPEND AMGX_EXAMPLES_SRCS
+    ex1p.cpp
+    )
+endif()
+
+set(AMGX_JSON_FILES amg_pcg.json multi_gs.json precon.json)
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add targets to copy *.json files from the source directory
+foreach(JSON_FILE ${AMGX_JSON_FILES})
+  add_custom_command(OUTPUT ${JSON_FILE}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/${JSON_FILE} ${JSON_FILE}
+    COMMENT "copy ${JSON_FILE}")
+endforeach()
+add_custom_target(copy_amgx_json_files DEPENDS ${AMGX_JSON_FILES}
+  COMMENT "Copying AMGX example json files ...")
+
+# Add "test_amgx" target, see below.
+add_custom_target(test_amgx
+  ${CMAKE_CTEST_COMMAND} -R amgx USES_TERMINAL)
+
+# Add one executable per cpp file, adding "amgx_" as prefix. Sets
+# "copy_amgx_json_files" as a prerequisite for the given examples. Also, sets
+# "test_amgx" as a target that depends on the given examples.
+set(PFX amgx_)
+add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
+
+# Testing.
+# The AMGX tests can be run separately using the target "test_amgx"
+# which builds the examples and runs:
+#   ctest -R amgx
+
+# Command line options for the tests.
+# Example 1/1p:
+set(EX1_TEST_OPTS)
+set(EX1P_TEST_OPTS)
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on NVIDIA's multigrid library AmgX.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_AMGX = YES", see the top-level INSTALL file for details (version
+2.1 of AmgX is recommended, though older versions may work too.
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.
@@ -17,7 +17,7 @@
            "interpolator": "D2",
            "max_row_sum" : 0.9,
            "strength_threshold" : 0.25,
-            "max_iters": 1,
+            "max_iters": 2,
            "monitor_residual": 0,
            "store_res_history": 0,
            "scope": "amg",
@@ -1,50 +1,15 @@
 //                                MFEM Example 1
+//                              AmgX Modification
 //
 // Compile with: make ex1
 //
-// Sample runs:  ex1 -m ../../data/square-disc.mesh
-//               ex1 -m ../../data/star.mesh
-//               ex1 -m ../../data/star-mixed.mesh
-//               ex1 -m ../../data/escher.mesh
-//               ex1 -m ../../data/fichera.mesh
-//               ex1 -m ../../data/fichera-mixed.mesh
-//               ex1 -m ../../data/toroid-wedge.mesh
-//               ex1 -m ../../data/periodic-annulus-sector.msh
-//               ex1 -m ../../data/periodic-torus-sector.msh
-//               ex1 -m ../../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/star-surf.mesh
-//               ex1 -m ../../data/square-disc-surf.mesh
-//               ex1 -m ../../data/inline-segment.mesh
-//               ex1 -m ../../data/amr-quad.mesh
-//               ex1 -m ../../data/amr-hex.mesh
-//               ex1 -m ../../data/fichera-amr.mesh
-//               ex1 -m ../../data/mobius-strip.mesh
-//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
-//
 // AmgX sample runs:
-//               ./ex1 --amgx-file multi_gs.json --amgx-solver
-//               ./ex1 --amgx-file precon.json --amgx-preconditioner
-//               ./ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
-//               ./ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
+//               ex1
+//               ex1 -d cuda
+//               ex1 --amgx-file multi_gs.json --amgx-solver
+//               ex1 --amgx-file precon.json --amgx-preconditioner
+//               ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
+//               ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -77,8 +42,9 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
+   bool amgx_lib = true;
   bool amgx_solver = true;
-   const char* amgx_json_file = ""; // jason file for amgx
+   const char* amgx_json_file = ""; // JSON file for AmgX

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -90,11 +56,12 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
+                  "--no-amgx-lib", "Use AmgX in example.");
   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
   args.AddOption(&amgx_solver, "--amgx-solver", "--amgx-solver",
-                  "--amgx-preconditioner",
-                  "--amgx-preconditioner",
+                  "--amgx-preconditioner", "--amgx-preconditioner",
                  "Configure AMGX as solver or preconditioner.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
@@ -109,9 +76,6 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   MFEM_VERIFY(!pa && strcmp(amgx_json_file,"") != 0,
-               "An AmgX json file is needed for this example \n");
-
   // 2. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
   Device device(device_config);
@@ -208,10 +172,29 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
-   if (!pa)
+   if (pa)
+   {
+      // Jacobi preconditioning in partial assembly mode
+      if (UsesTensorBasis(fespace))
+      {
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+      }
+      else
+      {
+         CG(*A, B, X, 1, 400, 1e-12, 0.0);
+      }
+   }
+   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
+   {
+      bool amgx_verbose = false;
+      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
+      amgx.SetOperator(*A.As<SparseMatrix>());
+      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
+   }
+   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
   {
      AmgXSolver amgx;
-
      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
      amgx.InitSerial();
      amgx.SetOperator(*A.As<SparseMatrix>());
@@ -224,19 +207,20 @@ int main(int argc, char *argv[])
      {
         PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
      }
-
   }
-   else // Jacobi preconditioning in partial assembly mode
+   else
   {
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
+#ifndef MFEM_USE_SUITESPARSE
+      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+      GSSmoother M((SparseMatrix&)(*A));
+      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(*A);
+      umf_solver.Mult(B, X);
+#endif
   }

   // 12. Recover the solution as a finite element grid function.
@@ -1,45 +1,13 @@
 //                       MFEM Example 1 - Parallel Version
+//                              AmgX Modification
 //
 // Compile with: make ex1p
 //
-// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../../data/star.mesh
-//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -m ../../data/beam-tet.mesh -pa -d ceed-cpu
-//
 // AmgX sample runs:
-//
-//               mpirun -n 40 ./ex1p --amgx-file amg_pcg.json
-//               lrun -n 4 ./ex1p --amgx-file amg_pcg.json --amgx-mpi-gpu-exclusive
+//               mpirun -np 4 ex1p
+//               mpirun -np 4 ex1p -d cuda
+//               mpirun -np 10 ex1p --amgx-file amg_pcg.json --amgx-mpi-teams
+//               mpirun -np 4 ex1p --amgx-file amg_pcg.json
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -78,8 +46,9 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   bool amgx_mpi_teams = true;
-   const char* amgx_json_file = ""; // jason file for amgx
+   bool amgx_lib = true;
+   bool amgx_mpi_teams = false;
+   const char* amgx_json_file = ""; // JSON file for AmgX
   int ndevices = 1;

   OptionsParser args(argc, argv);
@@ -92,17 +61,20 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
+                  "--no-amgx-lib", "Use AmgX in example.");
   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
   args.AddOption(&amgx_mpi_teams, "--amgx-mpi-teams", "--amgx-mpi-teams",
                  "--amgx-mpi-gpu-exclusive", "--amgx-mpi-gpu-exclusive",
-                  "Create MPI teams when using AMGX.");
+                  "Create MPI teams when using AmgX to load balance between ranks and GPUs.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&ndevices, "-nd","--nd","Number of GPU devices.");
+   args.AddOption(&ndevices, "-nd","--gpus-per-node-in-teams-mode",
+                  "Number of GPU devices per node (Only used if amgx_mpi_teams is true).");

   args.Parse();
   if (!args.Good())
@@ -117,9 +89,6 @@ int main(int argc, char *argv[])
   if (myid == 0)
   {
      args.PrintOptions(cout);
-
-      MFEM_VERIFY(!pa == true && strcmp(amgx_json_file,"") != 0,
-                  "An AmgX json file is needed for this example \n");
   }

   // 3. Enable hardware devices such as GPUs, and programming models such as
@@ -236,34 +205,11 @@ int main(int argc, char *argv[])

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
+   //     * If AmgX is available solve using amg preconditioner.
   //     * With partial assembly, use Jacobi smoothing, for now.
   Solver *prec = NULL;
-   if (!pa)
+   if (pa)
   {
-
-      AmgXSolver amgx;
-
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-
-      if (amgx_mpi_teams)
-      {
-         //Forms MPI teams to load balance between mpi ranks and gpus
-         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
-      }
-      else
-      {
-         //Assumes MPI == number of devices
-         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
-      }
-
-      amgx.SetOperator(*A.As<HypreParMatrix>());
-
-      amgx.Mult(B, X);
-
-   }
-   else
-   {
-
      if (UsesTensorBasis(fespace))
      {
         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
@@ -277,7 +223,60 @@ int main(int argc, char *argv[])
      cg.SetOperator(*A);
      cg.Mult(B, X);
      delete prec;
+   }
+   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
+   {
+      MFEM_VERIFY(!amgx_mpi_teams,
+                  "Please add JSON file to try AmgX with MPI teams mode");

+      bool amgx_verbose = false;
+      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
+                            amgx_verbose);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      if (prec) { cg.SetPreconditioner(*prec); }
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
+
+   }
+   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
+   {
+      AmgXSolver amgx;
+      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
+
+      if (amgx_mpi_teams)
+      {
+         // Forms MPI teams to load balance between MPI ranks and GPUs
+         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
+      }
+      else
+      {
+         // Assumes each MPI rank is paired with a GPU
+         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
+      }
+
+      amgx.SetOperator(*A.As<HypreParMatrix>());
+      amgx.Mult(B, X);
+
+      // Release MPI communicators and resources created by AmgX
+      amgx.Finalize();
+   }
+   else
+   {
+      prec = new HypreBoomerAMG;
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      if (prec) { cg.SetPreconditioner(*prec); }
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
   }

   // 14. Recover the parallel grid function corresponding to X. This is the
@@ -70,3 +70,6 @@ clean: clean-build
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f .logamgx refined.mesh sol.gf mesh.* sol.*
@@ -35,6 +35,7 @@
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
 //             * ex1 -pa -d ceed-cuda
+//             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
@@ -71,7 +72,6 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   bool amgx = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -83,8 +83,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
-                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -194,31 +192,7 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
-   if (pa)
-   {
-      // Jacobi preconditioning in partial assembly mode
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-   else if (amgx)
-   {
-#if defined(MFEM_USE_AMGX)
-      bool amgx_verbose = false;
-      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
-      amgx.SetOperator(*A.As<SparseMatrix>());
-      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
-#else
-      mfem_error("MFEM not configured with AMGX \n");
-#endif
-   }
-   else
+   if (!pa)
   {
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
@@ -232,6 +206,18 @@ int main(int argc, char *argv[])
      umf_solver.Mult(B, X);
 #endif
   }
+   else // Jacobi preconditioning in partial assembly mode
+   {
+      if (UsesTensorBasis(fespace))
+      {
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+      }
+      else
+      {
+         CG(*A, B, X, 1, 400, 1e-12, 0.0);
+      }
+   }

   // 12. Recover the solution as a finite element grid function.
   a.RecoverFEMSolution(X, b, x);
@@ -72,6 +72,7 @@ int main(int argc, char *argv[])
   int seed = 75;
   bool slu_solver  = false;
   bool sp_solver = false;
+   bool cpardiso_solver = false;
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -95,6 +96,10 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
                  "--no-strumpack", "Use the STRUMPACK Solver.");
+#endif
+#ifdef MFEM_USE_MKL_CPARDISO
+   args.AddOption(&cpardiso_solver, "-cpardiso", "--cpardiso", "-no-cpardiso",
+                  "--no-cpardiso", "Use the MKL CPardiso Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -236,7 +241,7 @@ int main(int argc, char *argv[])
   //    preconditioner for A to be used within the solver. Set the matrices
   //    which define the generalized eigenproblem A x = lambda M x.
   Solver * precond = NULL;
-   if (!slu_solver && !sp_solver)
+   if (!slu_solver && !sp_solver && !cpardiso_solver)
   {
      HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
      amg->SetPrintLevel(0);
@@ -268,10 +273,19 @@ int main(int argc, char *argv[])
         strumpack->SetFromCommandLine();
         precond = strumpack;
      }
+#endif
+#ifdef MFEM_USE_MKL_CPARDISO
+      if (cpardiso_solver)
+      {
+         auto cpardiso = new CPardisoSolver(A->GetComm());
+         cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
+         cpardiso->SetPrintLevel(1);
+         cpardiso->SetOperator(*A);
+         precond = cpardiso;
+      }
 #endif
   }

-
   HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
   lobpcg->SetNumModes(nev);
   lobpcg->SetRandomSeed(seed);
@@ -8,6 +8,7 @@
 //    ex19 -m ../data/beam-hex.mesh
 //    ex19 -m ../data/beam-tet.mesh
 //    ex19 -m ../data/beam-wedge.mesh
+//    ex19 -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -96,7 +97,7 @@ protected:
   Array<FiniteElementSpace *> spaces;

   // Offsets for extracting block vector segments
-   Array<int> &block_offsets;
+   Array<int> &block_trueOffsets;

   // Jacobian for block access
   BlockOperator *jacobian;
@@ -152,7 +153,7 @@ protected:
   Coefficient &mu;

   // Block offsets for variable access
-   Array<int> &block_offsets;
+   Array<int> &block_trueOffsets;

 public:
   RubberOperator(Array<FiniteElementSpace *> &fes, Array<Array<int> *>&ess_bdr,
@@ -246,8 +247,8 @@ int main(int argc, char *argv[])
   spaces[0] = &R_space;
   spaces[1] = &W_space;

-   int R_size = R_space.GetVSize();
-   int W_size = W_space.GetVSize();
+   int R_size = R_space.GetTrueVSize();
+   int W_size = W_space.GetTrueVSize();

   // 6. Define the Dirichlet conditions (set to boundary attribute 1 and 2)
   Array<Array<int> *> ess_bdr(2);
@@ -271,13 +272,13 @@ int main(int argc, char *argv[])
   std::cout << "***********************************************************\n";

   // 8. Define the block structure of the solution vector (u then p)
-   Array<int> block_offsets(3);
-   block_offsets[0] = 0;
-   block_offsets[1] = R_space.GetVSize();
-   block_offsets[2] = W_space.GetVSize();
-   block_offsets.PartialSum();
+   Array<int> block_trueOffsets(3);
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = R_space.GetTrueVSize();
+   block_trueOffsets[2] = W_space.GetTrueVSize();
+   block_trueOffsets.PartialSum();

-   BlockVector xp(block_offsets);
+   BlockVector xp(block_trueOffsets);

   // 9. Define grid functions for the current configuration, reference
   //    configuration, final deformation, and pressure
@@ -286,8 +287,8 @@ int main(int argc, char *argv[])
   GridFunction x_def(&R_space);
   GridFunction p_gf(&W_space);

-   x_gf.MakeRef(&R_space, xp.GetBlock(0), 0);
-   p_gf.MakeRef(&W_space, xp.GetBlock(1), 0);
+   x_gf.MakeTRef(&R_space, xp.GetBlock(0), 0);
+   p_gf.MakeTRef(&W_space, xp.GetBlock(1), 0);

   VectorFunctionCoefficient deform(dim, InitialDeformation);
   VectorFunctionCoefficient refconfig(dim, ReferenceConfiguration);
@@ -296,14 +297,19 @@ int main(int argc, char *argv[])
   x_ref.ProjectCoefficient(refconfig);
   p_gf = 0.0;

+   x_gf.SetTrueVector();
+   p_gf.SetTrueVector();
+
   // 10. Initialize the incompressible neo-Hookean operator
-   RubberOperator oper(spaces, ess_bdr, block_offsets,
+   RubberOperator oper(spaces, ess_bdr, block_trueOffsets,
                       newton_rel_tol, newton_abs_tol, newton_iter, c_mu);

   // 11. Solve the Newton system
   oper.Solve(xp);

   // 12. Compute the final deformation
+   x_gf.SetFromTrueVector();
+   p_gf.SetFromTrueVector();
   subtract(x_gf, x_ref, x_def);

   // 13. Visualize the results if requested
@@ -349,7 +355,7 @@ int main(int argc, char *argv[])
 JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
                                               SparseMatrix &mass,
                                               Array<int> &offsets)
-   : Solver(offsets[2]), block_offsets(offsets), pressure_mass(&mass)
+   : Solver(offsets[2]), block_trueOffsets(offsets), pressure_mass(&mass)
 {
   fes.Copy(spaces);

@@ -381,18 +387,18 @@ JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_offsets[0],
-                  block_offsets[1]-block_offsets[0]);
-   Vector pres_in(k.GetData() + block_offsets[1],
-                  block_offsets[2]-block_offsets[1]);
+   Vector disp_in(k.GetData() + block_trueOffsets[0],
+                  block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_in(k.GetData() + block_trueOffsets[1],
+                  block_trueOffsets[2]-block_trueOffsets[1]);

-   Vector disp_out(y.GetData() + block_offsets[0],
-                   block_offsets[1]-block_offsets[0]);
-   Vector pres_out(y.GetData() + block_offsets[1],
-                   block_offsets[2]-block_offsets[1]);
+   Vector disp_out(y.GetData() + block_trueOffsets[0],
+                   block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out(y.GetData() + block_trueOffsets[1],
+                   block_trueOffsets[2]-block_trueOffsets[1]);

-   Vector temp(block_offsets[1]-block_offsets[0]);
-   Vector temp2(block_offsets[1]-block_offsets[0]);
+   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);

   // Perform the block elimination for the preconditioner
   mass_pcg->Mult(pres_in, pres_out);
@@ -447,9 +453,9 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
                               double abs_tol,
                               int iter,
                               Coefficient &c_mu)
-   : Operator(fes[0]->GetVSize() + fes[1]->GetVSize()),
+   : Operator(fes[0]->GetTrueVSize() + fes[1]->GetTrueVSize()),
     newton_solver(), newton_monitor("Newton", 1),
-     j_monitor("  GMRES", 3), mu(c_mu), block_offsets(offsets)
+     j_monitor("  GMRES", 3), mu(c_mu), block_trueOffsets(offsets)
 {
   Array<Vector *> rhs(2);
   rhs = NULL; // Set all entries in the array
@@ -471,12 +477,16 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
   a->AddDomainIntegrator(new MassIntegrator(one));
   a->Assemble();
   a->Finalize();
+
+   OperatorPtr op;
+   Array<int> p_ess_tdofs;
+   a->FormSystemMatrix(p_ess_tdofs, op);
   pressure_mass = a->LoseMat();
   delete a;

   // Initialize the Jacobian preconditioner
   JacobianPreconditioner *jac_prec =
-      new JacobianPreconditioner(fes, *pressure_mass, block_offsets);
+      new JacobianPreconditioner(fes, *pressure_mass, block_trueOffsets);
   j_prec = jac_prec;

   // Set up the Jacobian solver
@@ -8,6 +8,7 @@
 //    mpirun -np 2 ex19p -m ../data/beam-hex.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-tet.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-wedge.mesh
+//    mpirun -np 2 ex19p -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -196,10 +197,8 @@ void InitialDeformation(const Vector &x, Vector &y);
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+   MPI_Session mpi;
+   const int myid = mpi.WorldRank();

   // 2. Parse command-line options
   const char *mesh_file = "../data/beam-tet.mesh";
@@ -239,7 +238,6 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
-      MPI_Finalize();
      return 1;
   }
   if (myid == 0)
@@ -399,8 +397,6 @@ int main(int argc, char *argv[])
   // 19. Free the used memory
   delete pmesh;

-   MPI_Finalize();
-
   return 0;
 }

@@ -474,7 +470,11 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
   {
      HypreBoomerAMG *stiff_prec_amg = new HypreBoomerAMG();
      stiff_prec_amg->SetPrintLevel(0);
-      stiff_prec_amg->SetElasticityOptions(spaces[0]);
+
+      if (!spaces[0]->GetParMesh()->Nonconforming())
+      {
+         stiff_prec_amg->SetElasticityOptions(spaces[0]);
+      }

      stiff_prec = stiff_prec_amg;

@@ -33,6 +33,7 @@
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
+//             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
@@ -73,7 +74,6 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   bool amgx = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -85,8 +85,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
-                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -221,7 +219,6 @@ int main(int argc, char *argv[])

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * If AmgX is available solve using amg preconditioner.
   //     * With partial assembly, use Jacobi smoothing, for now.
   Solver *prec = NULL;
   if (pa)
@@ -231,16 +228,6 @@ int main(int argc, char *argv[])
         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
      }
   }
-   else if (amgx)
-   {
-#if defined(MFEM_USE_AMGX)
-      bool amgx_verbose = false;
-      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
-                            amgx_verbose);
-#else
-      mfem_error("MFEM not configured with AMGX \n");
-#endif
-   }
   else
   {
      prec = new HypreBoomerAMG;
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Ginkgo high-performance linear algebra library
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM for solving nonlinear constrained optimization problems, including
@@ -48,6 +48,12 @@ endif
 ifeq ($(MFEM_USE_GINKGO),YES)
   SUBDIRS += ginkgo
 endif
+ifeq ($(MFEM_USE_AMGX),YES)
+   SUBDIRS += amgx
+endif
+ifeq ($(MFEM_USE_SUPERLU),YES)
+   SUBDIRS += superlu
+endif

 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
@@ -119,6 +125,11 @@ ex11p-test-superlu: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
 test-par-YES: ex11p-test-superlu
 endif
+ifeq ($(MFEM_USE_MKL_CPARDISO),YES)
+ex11p-test-cpardiso: ex11p
+	@$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso)
+test-par-YES: ex11p-test-cpardiso
+endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the PETSc suite.
@@ -410,21 +410,16 @@ int main(int argc, char *argv[])
   }

   // 12. Free the used memory.
-   if (!use_slepc)
-   {
-      delete lobpcg;
-   }
-   else
-   {
-      delete slepc;
-   }
+   delete lobpcg;
+   delete slepc;
   delete precond;
   delete M;
   delete A;
+   delete pA;
+   delete pM;
 #if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
   delete Arow;
 #endif
-
   delete fespace;
   if (order > 0)
   {
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Parallel Unstructured Mesh Infrastructure,
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               http://mfem.org
+                               https://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the SUNDIALS suite of time integration and
@@ -721,12 +721,14 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,

   if (nls_type == KINSOL)
   {
-      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_NONE, true);
+      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_LINESEARCH, true);
+      kinsolver->SetJFNK(true);
+      kinsolver->SetLSMaxIter(100);
      newton_solver = kinsolver;
      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(200);
      newton_solver->SetRelTol(rel_tol);
-      newton_solver->SetPrintLevel(0);
+      newton_solver->SetPrintLevel(1);
      kinsolver->SetMaxSetupCalls(4);
   }
   else
@@ -13,17 +13,27 @@
 //    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 7 -dt 0.005 -tf 9
 //    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 8 -dt 0.02 -tf 8 -o 2
 //
+// Device sample runs:
+//    ex9 -pa
+//    ex9 -ea
+//    ex9 -fa
+//    ex9 -pa -m ../../data/periodic-cube.mesh
+//    ex9 -pa -m ../../data/periodic-cube.mesh -d cuda
+//    ex9 -ea -m ../../data/periodic-cube.mesh -d cuda
+//    ex9 -fa -m ../../data/periodic-cube.mesh -d cuda
+//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of implicit
+//               and explicit ODE time integrators, the definition of periodic
+//               boundary conditions through periodic meshes, as well as the use
+//               of GLVis for persistent visualization of a time-evolving
+//               solution. The saving of time-dependent data files for external
+//               visualization with VisIt (visit.llnl.gov) and ParaView
+//               (paraview.org) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -53,6 +63,54 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

+class DG_Solver : public Solver
+{
+private:
+   SparseMatrix &M, &K, A;
+   GMRESSolver linear_solver;
+   BlockILU prec;
+   double dt;
+public:
+   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
+      : M(M_),
+        K(K_),
+        prec(fes.GetFE(0)->GetDof(),
+             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
+        dt(-1.0)
+   {
+      linear_solver.iterative_mode = false;
+      linear_solver.SetRelTol(1e-9);
+      linear_solver.SetAbsTol(0.0);
+      linear_solver.SetMaxIter(100);
+      linear_solver.SetPrintLevel(0);
+      linear_solver.SetPreconditioner(prec);
+   }
+
+   void SetTimeStep(double dt_)
+   {
+      if (dt_ != dt)
+      {
+         dt = dt_;
+         // Form operator A = M - dt*K
+         A = K;
+         A *= -dt;
+         A += M;
+
+         // this will also call SetOperator on the preconditioner
+         linear_solver.SetOperator(A);
+      }
+   }
+
+   void SetOperator(const Operator &op)
+   {
+      linear_solver.SetOperator(op);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      linear_solver.Mult(x, y);
+   }
+};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -62,19 +120,21 @@ Vector bb_min, bb_max;
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   SparseMatrix &M, &K;
+   BilinearForm &M, &K;
   const Vector &b;
-   DSmoother M_prec;
+   Solver *M_prec;
   CGSolver M_solver;
+   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);
+   FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution() { }
+   virtual ~FE_Evolution();
 };


@@ -85,11 +145,16 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
   int ref_levels = 2;
   int order = 3;
-   int ode_solver_type = 4;
+   bool pa = false;
+   bool ea = false;
+   bool fa = false;
+   const char *device_config = "cpu";
+   int ode_solver_type = 7;
   double t_final = 10.0;
   double dt = 0.01;
-   bool visualization = true;
+   bool visualization = false;
   bool visit = false;
+   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;

@@ -108,6 +173,14 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
+                  "--no-element-assembly", "Enable Element Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver:\n\t"
                  "1 - Forward Euler,\n\t"
@@ -128,6 +201,9 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
+                  "--no-paraview-datafiles",
+                  "Save data files for ParaView (paraview.org) visualization.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -147,6 +223,9 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

+   Device device(device_config);
+   device.Print();
+
   // 2. Read the mesh from the given mesh file. We can handle geometrically
   //    periodic meshes in this code.
   Mesh mesh(mesh_file, 1, 1);
@@ -168,7 +247,7 @@ int main(int argc, char *argv[])

   // 4. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim);
+   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
   FiniteElementSpace fes(&mesh, &fec);

   cout << "Number of unknowns: " << fes.GetVSize() << endl;
@@ -181,8 +260,23 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   BilinearForm m(&fes);
-   m.AddDomainIntegrator(new MassIntegrator);
   BilinearForm k(&fes);
+   if (pa)
+   {
+      m.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   }
+   else if (ea)
+   {
+      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
+      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
+   }
+   else if (fa)
+   {
+      m.SetAssemblyLevel(AssemblyLevel::FULL);
+      k.SetAssemblyLevel(AssemblyLevel::FULL);
+   }
+   m.AddDomainIntegrator(new MassIntegrator);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -194,11 +288,11 @@ int main(int argc, char *argv[])
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

   m.Assemble();
-   m.Finalize();
   int skip_zeros = 0;
   k.Assemble(skip_zeros);
-   k.Finalize(skip_zeros);
   b.Assemble();
+   m.Finalize();
+   k.Finalize(skip_zeros);

   // 6. Define the initial conditions, save the corresponding grid function to
   //    a file and (optionally) save data in the VisIt format and initialize
@@ -239,6 +333,20 @@ int main(int argc, char *argv[])
      dc->Save();
   }

+   ParaViewDataCollection *pd = NULL;
+   if (paraview)
+   {
+      pd = new ParaViewDataCollection("Example9", &mesh);
+      pd->SetPrefixPath("ParaView");
+      pd->RegisterField("solution", &u);
+      pd->SetLevelsOfDetail(order);
+      pd->SetDataFormat(VTKFormat::BINARY);
+      pd->SetHighOrderOutput(true);
+      pd->SetCycle(0);
+      pd->SetTime(0.0);
+      pd->Save();
+   }
+
   socketstream sout;
   if (visualization)
   {
@@ -265,7 +373,7 @@ int main(int argc, char *argv[])

   // 7. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(m.SpMat(), k.SpMat(), b);
+   FE_Evolution adv(m, k, b);

   double t = 0.0;
   adv.SetTime(t);
@@ -289,12 +397,18 @@ int main(int argc, char *argv[])
         cvode->UseSundialsLinearSolver();
         ode_solver = cvode; break;
      case 8:
+         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
+         arkode->Init(adv);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         arkode->SetOrder(4);
+         ode_solver = arkode; break;
      case 9:
         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         arkode->SetERKTableNum(FEHLBERG_13_7_8);
         ode_solver = arkode; break;
   }

@@ -329,6 +443,13 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
+
+         if (paraview)
+         {
+            pd->SetCycle(ti);
+            pd->SetTime(t);
+            pd->Save();
+         }
      }
   }

@@ -342,6 +463,7 @@ int main(int argc, char *argv[])

   // 10. Free the used memory.
   delete ode_solver;
+   delete pd;
   delete dc;

   return 0;
@@ -349,12 +471,23 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
+FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
-
+   Array<int> ess_tdof_list;
+   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
+   {
+      M_prec = new DSmoother(M.SpMat());
+      M_solver.SetOperator(M.SpMat());
+      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
+   }
+   else
+   {
+      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
+      M_solver.SetOperator(M);
+      dg_solver = NULL;
+   }
+   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -370,6 +503,21 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
   M_solver.Mult(z, y);
 }

+void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+{
+   MFEM_VERIFY(dg_solver != NULL,
+               "Implicit time integration is not supported with partial assembly");
+   K.Mult(x, z);
+   z += b;
+   dg_solver->SetTimeStep(dt);
+   dg_solver->Mult(z, k);
+}
+
+FE_Evolution::~FE_Evolution()
+{
+   delete M_prec;
+   delete dg_solver;
+}

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -13,17 +13,28 @@
 //    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 7 -dt 0.0025 -tf 9
 //    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 8 -dt 0.01 -tf 8 -o 2
 //
+// Device sample runs:
+//    mpirun -np 4 ex9p -pa
+//    mpirun -np 4 ex9p -ea
+//    mpirun -np 4 ex9p -fa
+//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh
+//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh -d cuda
+//    mpirun -np 4 ex9p -ea -m ../../data/periodic-cube.mesh -d cuda
+//    mpirun -np 4 ex9p -fa -m ../../data/periodic-cube.mesh -d cuda
+//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of implicit
+//               and explicit ODE time integrators, the definition of periodic
+//               boundary conditions through periodic meshes, as well as the use
+//               of GLVis for persistent visualization of a time-evolving
+//               solution. Saving of time-dependent data files for visualization
+//               with VisIt (visit.llnl.gov) and ParaView (paraview.org), as
+//               well as the optional saving with ADIOS2 (adios2.readthedocs.io)
+//               are also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -52,6 +63,66 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

+class DG_Solver : public Solver
+{
+private:
+   HypreParMatrix &M, &K;
+   SparseMatrix M_diag;
+   HypreParMatrix *A;
+   GMRESSolver linear_solver;
+   BlockILU prec;
+   double dt;
+public:
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
+      : M(M_),
+        K(K_),
+        A(NULL),
+        linear_solver(M.GetComm()),
+        prec(fes.GetFE(0)->GetDof(),
+             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
+        dt(-1.0)
+   {
+      linear_solver.iterative_mode = false;
+      linear_solver.SetRelTol(1e-9);
+      linear_solver.SetAbsTol(0.0);
+      linear_solver.SetMaxIter(100);
+      linear_solver.SetPrintLevel(0);
+      linear_solver.SetPreconditioner(prec);
+
+      M.GetDiag(M_diag);
+   }
+
+   void SetTimeStep(double dt_)
+   {
+      if (dt_ != dt)
+      {
+         dt = dt_;
+         // Form operator A = M - dt*K
+         delete A;
+         A = Add(-dt, K, 0.0, K);
+         SparseMatrix A_diag;
+         A->GetDiag(A_diag);
+         A_diag.Add(1.0, M_diag);
+         // this will also call SetOperator on the preconditioner
+         linear_solver.SetOperator(*A);
+      }
+   }
+
+   void SetOperator(const Operator &op)
+   {
+      linear_solver.SetOperator(op);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      linear_solver.Mult(x, y);
+   }
+
+   ~DG_Solver()
+   {
+      delete A;
+   }
+};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -61,19 +132,21 @@ Vector bb_min, bb_max;
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   HypreParMatrix &M, &K;
+   OperatorHandle M, K;
   const Vector &b;
-   HypreSmoother M_prec;
+   Solver *M_prec;
   CGSolver M_solver;
+   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);
+   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution() { }
+   virtual ~FE_Evolution();
 };


@@ -91,11 +164,17 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 0;
   int order = 3;
-   int ode_solver_type = 4;
+   bool pa = false;
+   bool ea = false;
+   bool fa = false;
+   const char *device_config = "cpu";
+   int ode_solver_type = 7;
   double t_final = 10.0;
   double dt = 0.01;
-   bool visualization = true;
+   bool visualization = false;
   bool visit = false;
+   bool paraview = false;
+   bool adios2 = false;
   bool binary = false;
   int vis_steps = 5;

@@ -116,6 +195,14 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly in parallel.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
+   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
+                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
+                  "--no-element-assembly", "Enable Element Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver:\n\t"
                  "1 - Forward Euler,\n\t"
@@ -136,6 +223,12 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
+                  "--no-paraview-datafiles",
+                  "Save data files for ParaView (paraview.org) visualization.");
+   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
+                  "--no-adios2-streams",
+                  "Save data using adios2 streams.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -155,6 +248,7 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
+
   // check for vaild ODE solver option
   if (ode_solver_type < 1 || ode_solver_type > 9)
   {
@@ -166,6 +260,9 @@ int main(int argc, char *argv[])
      return 3;
   }

+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle geometrically periodic meshes in this code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -197,7 +294,7 @@ int main(int argc, char *argv[])

   // 6. Define the parallel discontinuous DG finite element space on the
   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim);
+   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);

   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
@@ -214,8 +311,24 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   ParBilinearForm *m = new ParBilinearForm(fes);
-   m->AddDomainIntegrator(new MassIntegrator);
   ParBilinearForm *k = new ParBilinearForm(fes);
+   if (pa)
+   {
+      m->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      k->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   }
+   else if (ea)
+   {
+      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
+      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
+   }
+   else if (fa)
+   {
+      m->SetAssemblyLevel(AssemblyLevel::FULL);
+      k->SetAssemblyLevel(AssemblyLevel::FULL);
+   }
+
+   m->AddDomainIntegrator(new MassIntegrator);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -226,15 +339,13 @@ int main(int argc, char *argv[])
   b->AddBdrFaceIntegrator(
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

-   m->Assemble();
-   m->Finalize();
   int skip_zeros = 0;
+   m->Assemble();
   k->Assemble(skip_zeros);
-   k->Finalize(skip_zeros);
   b->Assemble();
+   m->Finalize();
+   k->Finalize(skip_zeros);

-   HypreParMatrix *M = m->ParallelAssemble();
-   HypreParMatrix *K = k->ParallelAssemble();
   HypreParVector *B = b->ParallelAssemble();

   // 8. Define the initial conditions, save the corresponding grid function to
@@ -273,6 +384,8 @@ int main(int argc, char *argv[])
      {
         dc = new VisItDataCollection("Example9-Parallel", pmesh);
         dc->SetPrecision(precision);
+         // To save the mesh using MFEM's parallel mesh format:
+         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
      }
      dc->RegisterField("solution", u);
      dc->SetCycle(0);
@@ -280,6 +393,41 @@ int main(int argc, char *argv[])
      dc->Save();
   }

+   ParaViewDataCollection *pd = NULL;
+   if (paraview)
+   {
+      pd = new ParaViewDataCollection("Example9P", pmesh);
+      pd->SetPrefixPath("ParaView");
+      pd->RegisterField("solution", u);
+      pd->SetLevelsOfDetail(order);
+      pd->SetDataFormat(VTKFormat::BINARY);
+      pd->SetHighOrderOutput(true);
+      pd->SetCycle(0);
+      pd->SetTime(0.0);
+      pd->Save();
+   }
+
+   // Optionally output a BP (binary pack) file using ADIOS2. This can be
+   // visualized with the ParaView VTX reader.
+#ifdef MFEM_USE_ADIOS2
+   ADIOS2DataCollection *adios2_dc = NULL;
+   if (adios2)
+   {
+      std::string postfix(mesh_file);
+      postfix.erase(0, std::string("../data/").size() );
+      postfix += "_o" + std::to_string(order);
+      const std::string collection_name = "ex9-p-" + postfix + ".bp";
+
+      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
+      // output data substreams are half the number of mpi processes
+      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
+      adios2_dc->RegisterField("solution", u);
+      adios2_dc->SetCycle(0);
+      adios2_dc->SetTime(0.0);
+      adios2_dc->Save();
+   }
+#endif
+
   socketstream sout;
   if (visualization)
   {
@@ -312,7 +460,7 @@ int main(int argc, char *argv[])

   // 9. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(*M, *K, *B);
+   FE_Evolution adv(*m, *k, *B);

   double t = 0.0;
   adv.SetTime(t);
@@ -384,6 +532,23 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
+
+         if (paraview)
+         {
+            pd->SetCycle(ti);
+            pd->SetTime(t);
+            pd->Save();
+         }
+
+#ifdef MFEM_USE_ADIOS2
+         // transient solutions can be visualized with ParaView
+         if (adios2)
+         {
+            adios2_dc->SetCycle(ti);
+            adios2_dc->SetTime(t);
+            adios2_dc->Save();
+         }
+#endif
      }
   }

@@ -403,13 +568,18 @@ int main(int argc, char *argv[])
   delete u;
   delete B;
   delete b;
-   delete K;
   delete k;
-   delete M;
   delete m;
   delete fes;
   delete pmesh;
   delete ode_solver;
+   delete pd;
+#ifdef MFEM_USE_ADIOS2
+   if (adios2)
+   {
+      delete adios2_dc;
+   }
+#endif
   delete dc;

   MPI_Finalize();
@@ -418,15 +588,43 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
+FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
                           const Vector &_b)
   : TimeDependentOperator(_M.Height()),
-     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
+     b(_b),
+     M_solver(_M.ParFESpace()->GetComm()),
+     z(_M.Height())
 {
-   M_prec.SetType(HypreSmoother::Jacobi);
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
+   {
+      M.Reset(_M.ParallelAssemble(), true);
+      K.Reset(_K.ParallelAssemble(), true);
+   }
+   else
+   {
+      M.Reset(&_M, false);
+      K.Reset(&_K, false);
+   }

+   M_solver.SetOperator(*M);
+
+   Array<int> ess_tdof_list;
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
+   {
+      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
+      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
+      HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
+      M_prec = hypre_prec;
+
+      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
+   }
+   else
+   {
+      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
+      dg_solver = NULL;
+   }
+
+   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -434,14 +632,28 @@ FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
   M_solver.SetPrintLevel(0);
 }

+void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
+{
+   K->Mult(x, z);
+   z += b;
+   dg_solver->SetTimeStep(dt);
+   dg_solver->Mult(z, k);
+}
+
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
+   K->Mult(x, z);
   z += b;
   M_solver.Mult(z, y);
 }

+FE_Evolution::~FE_Evolution()
+{
+   delete M_prec;
+   delete dg_solver;
+}
+

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -0,0 +1,62 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(SUPERLU_EXAMPLES_SRCS)
+if (MFEM_USE_MPI)
+  list(APPEND SUPERLU_EXAMPLES_SRCS
+    ex1p.cpp
+    )
+endif()
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_superlu" target, see below.
+add_custom_target(test_superlu
+  ${CMAKE_CTEST_COMMAND} -R superlu USES_TERMINAL)
+
+# Add one executable per cpp file, adding "superlu_" as prefix. Sets
+# "test_superlu" as a target that depends on the given examples.
+set(PFX superlu_)
+add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
+
+# Testing.
+# The SuperLU tests can be run separately using the target "test_superlu"
+# which builds the examples and runs:
+#   ctest -R superlu
+
+# Command line options for the tests.
+# Example 1: Test SuperLU on the simple Poisson problem
+set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
+set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -0,0 +1,321 @@
+//                       MFEM Example 1 - Parallel Version
+//                             SuperLU Modification
+//
+// Compile with: make ex1p
+//
+// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../../data/star.mesh
+//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
+//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
+//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               simple finite element discretization of the Laplace problem
+//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
+//               Specifically, we discretize using a FE space of the specified
+//               order, or if order < 1 using an isoparametric/isogeometric
+//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of mesh refinement, finite
+//               element grid functions, as well as linear and bilinear forms
+//               corresponding to the left-hand side and right-hand side of the
+//               discrete linear system. We also cover the explicit elimination
+//               of essential boundary conditions, static condensation, and the
+//               optional connection to the GLVis tool for visualization.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int order = 1;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   int slu_colperm = 4;
+   int slu_rowperm = 1;
+   int slu_iterref = 2;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&slu_colperm, "-cp", "--colperm",
+                  "SuperLU Column Permutation Method:  0-NATURAL, 1-MMD-ATA "
+                  "2-MMD_AT_PLUS_A, 3-COLAMD, 4-METIS_AT_PLUS_A, 5-PARMETIS "
+                  "6-ZOLTAN");
+   args.AddOption(&slu_rowperm, "-rp", "--rowperm",
+                  "SuperLU Row Permutation Method:  0-NOROWPERM, 1-LargeDiag");
+   args.AddOption(&slu_iterref, "-rp", "--rowperm",
+                  "SuperLU Iterative Refinement:  0-NOREFINE, 1-Single, "
+                  "2-Double, 3-Extra");
+
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement. We choose
+   //    'ref_levels' to be the largest number that gives a final mesh with no
+   //    more than 1,000 elements.
+   {
+      int ref_levels =
+         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh.UniformRefinement();
+      }
+   }
+
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+   {
+      int par_ref_levels = 2;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh.UniformRefinement();
+      }
+   }
+
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   bool delete_fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+      delete_fec = true;
+   }
+   else if (pmesh.GetNodes())
+   {
+      fec = pmesh.GetNodes()->OwnFEC();
+      delete_fec = false;
+      if (myid == 0)
+      {
+         cout << "Using isoparametric FEs: " << fec->Name() << endl;
+      }
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
+   }
+   ParFiniteElementSpace fespace(&pmesh, fec);
+   HYPRE_Int size = fespace.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+   }
+
+   // 8. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes from the mesh as essential
+   //    (Dirichlet) and converting them to a list of true dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system, which in this case is
+   //    (1,phi_i) where phi_i are the basis functions in fespace.
+   ParLinearForm b(&fespace);
+   ConstantCoefficient one(1.0);
+   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   b.Assemble();
+
+   // 10. Define the solution vector x as a parallel finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
+   ParBilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   // 12. Assemble the parallel bilinear form and the corresponding linear
+   //     system, applying any necessary transformations such as: parallel
+   //     assembly, eliminating boundary conditions, applying conforming
+   //     constraints for non-conforming AMR, static condensation, etc.
+   a.Assemble();
+
+   OperatorPtr A;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+   // 13. Solve the linear system A X = B utilizing SuperLU.
+   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD);
+   Operator *SLU_A = new SuperLURowLocMatrix(*A.As<HypreParMatrix>());
+   superlu->SetPrintStatistics(true);
+   superlu->SetSymmetricPattern(false);
+
+   if (slu_colperm == 0)
+   {
+      superlu->SetColumnPermutation(superlu::NATURAL);
+   }
+   else if (slu_colperm == 1)
+   {
+      superlu->SetColumnPermutation(superlu::MMD_ATA);
+   }
+   else if (slu_colperm == 2)
+   {
+      superlu->SetColumnPermutation(superlu::MMD_AT_PLUS_A);
+   }
+   else if (slu_colperm == 3)
+   {
+      superlu->SetColumnPermutation(superlu::COLAMD);
+   }
+   else if (slu_colperm == 4)
+   {
+      superlu->SetColumnPermutation(superlu::METIS_AT_PLUS_A);
+   }
+   else if (slu_colperm == 5)
+   {
+      superlu->SetColumnPermutation(superlu::PARMETIS);
+   }
+   else if (slu_colperm == 6)
+   {
+      superlu->SetColumnPermutation(superlu::ZOLTAN);
+   }
+
+   if (slu_rowperm == 0)
+   {
+      superlu->SetRowPermutation(superlu::NOROWPERM);
+   }
+   else if (slu_rowperm == 1)
+   {
+#ifdef MFEM_USE_SUPERLU5
+      superlu->SetRowPermutation(superlu::LargeDiag);
+#else
+      superlu->SetRowPermutation(superlu::LargeDiag_MC64);
+#endif
+   }
+
+   if (slu_iterref == 0)
+   {
+      superlu->SetIterativeRefine(superlu::NOREFINE);
+   }
+   else if (slu_iterref == 1)
+   {
+      superlu->SetIterativeRefine(superlu::SLU_SINGLE);
+   }
+   else if (slu_iterref == 2)
+   {
+      superlu->SetIterativeRefine(superlu::SLU_DOUBLE);
+   }
+   else if (slu_iterref == 3)
+   {
+      superlu->SetIterativeRefine(superlu::SLU_EXTRA);
+   }
+
+   superlu->SetOperator(*SLU_A);
+   superlu->SetPrintStatistics(true);
+   superlu->Mult(B, X);
+
+   // 14. Recover the parallel grid function corresponding to X. This is the
+   //     local finite element solution on each processor.
+   a.RecoverFEMSolution(X, b, x);
+
+   // 15. Save the refined mesh and the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   {
+      ostringstream mesh_name, sol_name;
+      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      pmesh.Print(mesh_ofs);
+
+      ofstream sol_ofs(sol_name.str().c_str());
+      sol_ofs.precision(8);
+      x.Save(sol_ofs);
+   }
+
+   // 16. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << x << flush;
+   }
+
+   // 17. Free the used memory.
+   if (delete_fec)
+   {
+      delete fec;
+   }
+   MPI_Finalize();
+
+   return 0;
+}
@@ -0,0 +1,79 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/superlu/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = 
+PAR_EXAMPLES = ex1p
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_SUPERLU),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with SuperLU)
+endif
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+PARALLEL_NAME := Parallel SuperLU example
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
+
+# Testing: Specific execution options:
+# Example 1: Test SuperLU on the simple poisson problem
+EX1_COMMON_ARGS := -m ../../data/star.mesh
+EX1P_ARGS := $(EX1_COMMON_ARGS)
+ex1p-test-par: ex1p
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX1P_ARGS))
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f mesh.* sol.*
@@ -17,6 +17,7 @@ set(SRCS
  bilininteg_convection_ea.cpp
  bilininteg_dgtrace_pa.cpp
  bilininteg_dgtrace_ea.cpp
+  bilininteg_diffusion_mf.cpp
  bilininteg_diffusion_pa.cpp
  bilininteg_diffusion_ea.cpp
  bilininteg_divergence.cpp
@@ -24,11 +25,14 @@ set(SRCS
  bilininteg_hdiv.cpp
  bilininteg_vectorfe.cpp
  bilininteg_gradient.cpp
+  bilininteg_mass_mf.cpp
  bilininteg_mass_pa.cpp
  bilininteg_mass_ea.cpp
  bilininteg_transpose_ea.cpp
  bilininteg_vecdiffusion.cpp
+  bilininteg_vecdiffusion_mf.cpp
  bilininteg_vecmass.cpp
+  bilininteg_vecmass_mf.cpp
  coefficient.cpp
  complex_fem.cpp
  convergence.cpp
@@ -42,6 +46,9 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
+  libceed/ceed.cpp
+  libceed/diffusion.cpp
+  libceed/mass.cpp
  linearform.cpp
  lininteg.cpp
  multigrid.cpp
@@ -78,6 +85,9 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
+  libceed/ceed.hpp
+  libceed/diffusion.hpp
+  libceed/mass.hpp
  linearform.hpp
  lininteg.hpp
  multigrid.hpp
@@ -137,17 +147,6 @@ if (MFEM_USE_MPI)
    prestriction.hpp)
 endif()

-if (MFEM_USE_CEED)
-   list(APPEND SRCS
-     libceed/ceed.cpp
-     libceed/diffusion.cpp
-     libceed/mass.cpp)
-   list(APPEND HDRS
-     libceed/ceed.hpp
-     libceed/diffusion.hpp
-     libceed/mass.hpp)
-endif()
-
 convert_filenames_to_full_paths(SRCS)
 convert_filenames_to_full_paths(HDRS)

@@ -133,8 +133,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
         ext = new PABilinearFormExtension(this);
         break;
      case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
-         // ext = new MFBilinearFormExtension(this);
+         ext = new MFBilinearFormExtension(this);
         break;
      default:
         mfem_error("Unknown assembly level");
@@ -157,7 +157,8 @@ public:
   /// Set the desired assembly level.
   /** Valid choices are:

-       - AssemblyLevel::FULL  (default)
+       - AssemblyLevel::LEGACYFULL (default)
+       - AssemblyLevel::FULL
       - AssemblyLevel::PARTIAL
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE
@@ -415,9 +416,7 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
+       destructor is called. */
   template <typename OpType>
   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
                         OpType &A, Vector &X, Vector &B,
@@ -439,9 +438,7 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
+       destructor is called. */
   template <typename OpType>
   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
   {
@@ -759,7 +756,7 @@ public:
   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }

-   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /// Set the desired assembly level. The default is AssemblyLevel::LEGACYFULL.
   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

@@ -861,9 +858,7 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
+       destructor is called. */
   template <typename OpType>
   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
                                    const Array<int> &test_tdof_list, OpType &A)
@@ -893,9 +888,7 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
+       destructor is called. */
   template <typename OpType>
   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
                                    const Array<int> &test_tdof_list,
@@ -36,6 +36,206 @@ const Operator *BilinearFormExtension::GetRestriction() const
   return a->GetRestriction();
 }

+// Data and methods for partially-assembled bilinear forms
+MFBilinearFormExtension::MFBilinearFormExtension(BilinearForm *form)
+   : BilinearFormExtension(form),
+     trialFes(a->FESpace()),
+     testFes(a->FESpace())
+{
+   elem_restrict = NULL;
+   int_face_restrict_lex = NULL;
+   bdr_face_restrict_lex = NULL;
+}
+
+void MFBilinearFormExtension::Assemble()
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integratorCount = integrators.Size();
+   for (int i = 0; i < integratorCount; ++i)
+   {
+      integrators[i]->AssembleMF(*a->FESpace());
+   }
+}
+
+void MFBilinearFormExtension::AssembleDiagonal(Vector &y) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+
+   const int iSz = integrators.Size();
+   if (elem_restrict && !DeviceCanUseCeed())
+   {
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AssembleDiagonalMF(localY);
+      }
+      const ElementRestriction* H1elem_restrict =
+         dynamic_cast<const ElementRestriction*>(elem_restrict);
+      if (H1elem_restrict)
+      {
+         H1elem_restrict->MultTransposeUnsigned(localY, y);
+      }
+      else
+      {
+         elem_restrict->MultTranspose(localY, y);
+      }
+   }
+   else
+   {
+      y.UseDevice(true); // typically this is a large vector, so store on device
+      y = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AssembleDiagonalMF(y);
+      }
+   }
+}
+
+void MFBilinearFormExtension::Update()
+{
+   FiniteElementSpace *fes = a->FESpace();
+   height = width = fes->GetVSize();
+   trialFes = fes;
+   testFes = fes;
+
+   elem_restrict = nullptr;
+   int_face_restrict_lex = nullptr;
+   bdr_face_restrict_lex = nullptr;
+}
+
+void MFBilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                               OperatorHandle &A)
+{
+   Operator *oper;
+   Operator::FormSystemOperator(ess_tdof_list, oper);
+   A.Reset(oper); // A will own oper
+}
+
+void MFBilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                               Vector &x, Vector &b,
+                                               OperatorHandle &A,
+                                               Vector &X, Vector &B,
+                                               int copy_interior)
+{
+   Operator *oper;
+   Operator::FormLinearSystem(ess_tdof_list, x, b, oper, X, B, copy_interior);
+   A.Reset(oper); // A will own oper
+}
+
+void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+
+   const int iSz = integrators.Size();
+   if (DeviceCanUseCeed() || !elem_restrict)
+   {
+      y.UseDevice(true); // typically this is a large vector, so store on device
+      y = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AddMultMF(x, y);
+      }
+   }
+   else
+   {
+      elem_restrict->Mult(x, localX);
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AddMultMF(localX, localY);
+      }
+      elem_restrict->MultTranspose(localY, y);
+   }
+
+   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
+   const int iFISz = intFaceIntegrators.Size();
+   if (int_face_restrict_lex && iFISz>0)
+   {
+      int_face_restrict_lex->Mult(x, faceIntX);
+      if (faceIntX.Size()>0)
+      {
+         faceIntY = 0.0;
+         for (int i = 0; i < iFISz; ++i)
+         {
+            intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
+         }
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
+      }
+   }
+
+   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
+   const int bFISz = bdrFaceIntegrators.Size();
+   if (bdr_face_restrict_lex && bFISz>0)
+   {
+      bdr_face_restrict_lex->Mult(x, faceBdrX);
+      if (faceBdrX.Size()>0)
+      {
+         faceBdrY = 0.0;
+         for (int i = 0; i < bFISz; ++i)
+         {
+            bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
+         }
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+      }
+   }
+}
+
+void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
+{
+   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int iSz = integrators.Size();
+   if (elem_restrict)
+   {
+      elem_restrict->Mult(x, localX);
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AddMultTransposeMF(localX, localY);
+      }
+      elem_restrict->MultTranspose(localY, y);
+   }
+   else
+   {
+      y.UseDevice(true);
+      y = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AddMultTransposeMF(x, y);
+      }
+   }
+
+   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
+   const int iFISz = intFaceIntegrators.Size();
+   if (int_face_restrict_lex && iFISz>0)
+   {
+      int_face_restrict_lex->Mult(x, faceIntX);
+      if (faceIntX.Size()>0)
+      {
+         faceIntY = 0.0;
+         for (int i = 0; i < iFISz; ++i)
+         {
+            intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
+         }
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
+      }
+   }
+
+   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
+   const int bFISz = bdrFaceIntegrators.Size();
+   if (bdr_face_restrict_lex && bFISz>0)
+   {
+      bdr_face_restrict_lex->Mult(x, faceBdrX);
+      if (faceBdrX.Size()>0)
+      {
+         faceBdrY = 0.0;
+         for (int i = 0; i < bFISz; ++i)
+         {
+            bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
+         }
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+      }
+   }
+}

 // Data and methods for partially-assembled bilinear forms
 PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
@@ -130,24 +130,31 @@ public:
   void MultTranspose(const Vector &x, Vector &y) const;
 };

-/// Data and methods for matrix-free bilinear forms NOT YET IMPLEMENTED.
+/// Data and methods for matrix-free bilinear forms
 class MFBilinearFormExtension : public BilinearFormExtension
 {
-public:
-   MFBilinearFormExtension(BilinearForm *form)
-      : BilinearFormExtension(form) { }
+protected:
+   const FiniteElementSpace *trialFes, *testFes; // Not owned
+   mutable Vector localX, localY;
+   mutable Vector faceIntX, faceIntY;
+   mutable Vector faceBdrX, faceBdrY;
+   const Operator *elem_restrict; // Not owned
+   const Operator *int_face_restrict_lex; // Not owned
+   const Operator *bdr_face_restrict_lex; // Not owned

-   /// TODO
-   void Assemble() {}
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
+public:
+   MFBilinearFormExtension(BilinearForm *form);
+
+   void Assemble();
+   void AssembleDiagonal(Vector &diag) const;
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
   void FormLinearSystem(const Array<int> &ess_tdof_list,
                         Vector &x, Vector &b,
                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0) {}
-   void Mult(const Vector &x, Vector &y) const {}
-   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
-   ~MFBilinearFormExtension() {}
+                         int copy_interior = 0);
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
+   void Update();
 };

 /// Class extending the MixedBilinearForm class to support different AssemblyLevels.
@@ -96,6 +96,30 @@ void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
               "   is not implemented for this class.");
 }

+void BilinearFormIntegrator::AssembleMF(const FiniteElementSpace &fes)
+{
+   mfem_error ("BilinearFormIntegrator::AssembleMF(...)\n"
+               "   is not implemented for this class.");
+}
+
+void BilinearFormIntegrator::AddMultMF(const Vector &, Vector &) const
+{
+   mfem_error ("BilinearFormIntegrator::AddMultMF(...)\n"
+               "   is not implemented for this class.");
+}
+
+void BilinearFormIntegrator::AddMultTransposeMF(const Vector &, Vector &) const
+{
+   mfem_error ("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
+               "   is not implemented for this class.");
+}
+
+void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
+{
+   mfem_error ("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
+               "   is not implemented for this class.");
+}
+
 void BilinearFormIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -133,8 +157,22 @@ void BilinearFormIntegrator::AssembleElementVector(
   const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
   Vector &elvect)
 {
-   mfem_error("BilinearFormIntegrator::AssembleElementVector\n"
-              "   is not implemented for this class.");
+   // Note: This default implementation is general but not efficient
+   DenseMatrix elmat;
+   AssembleElementMatrix(el, Tr, elmat);
+   elvect.SetSize(elmat.Height());
+   elmat.Mult(elfun, elvect);
+}
+
+void BilinearFormIntegrator::AssembleFaceVector(
+   const FiniteElement &el1, const FiniteElement &el2,
+   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
+{
+   // Note: This default implementation is general but not efficient
+   DenseMatrix elmat;
+   AssembleFaceMatrix(el1, el2, Tr, elmat);
+   elvect.SetSize(elmat.Height());
+   elmat.Mult(elfun, elvect);
 }


@@ -516,10 +554,12 @@ void DiffusionIntegrator::AssembleElementMatrix

 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), dshapedxt(nd,spaceDim), invdfdx(dim,spaceDim);
+   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd,dim);
   dshapedxt.SetSize(nd,spaceDim);
   invdfdx.SetSize(dim,spaceDim);
+   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   elmat.SetSize(nd);

@@ -537,7 +577,20 @@ void DiffusionIntegrator::AssembleElementMatrix
      // AdjugateJacobian = / adj(J),         if J is square
      //                    \ adj(J^t.J).J^t, otherwise
      Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
-      if (!MQ)
+      if (MQ)
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(dshapedxt, invdfdx, dshape);
+         AddMultABt(dshape, dshapedxt, elmat);
+      }
+      else if (VQ)
+      {
+         VQ->Eval(D, Trans, ip);
+         D *= w;
+         AddMultADAt(dshapedxt, D, elmat);
+      }
+      else
      {
         if (Q)
         {
@@ -545,13 +598,6 @@ void DiffusionIntegrator::AssembleElementMatrix
         }
         AddMult_a_AAt(w, dshapedxt, elmat);
      }
-      else
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(dshapedxt, invdfdx, dshape);
-         AddMultABt(dshape, dshapedxt, elmat);
-      }
   }
 }

@@ -570,12 +616,14 @@ void DiffusionIntegrator::AssembleElementMatrix2(
   DenseMatrix dshape(tr_nd, dim), dshapedxt(tr_nd, spaceDim);
   DenseMatrix te_dshape(te_nd, dim), te_dshapedxt(te_nd, spaceDim);
   DenseMatrix invdfdx(dim, spaceDim);
+   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(tr_nd, dim);
   dshapedxt.SetSize(tr_nd, spaceDim);
   te_dshape.SetSize(te_nd, dim);
   te_dshapedxt.SetSize(te_nd, spaceDim);
   invdfdx.SetSize(dim, spaceDim);
+   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   elmat.SetSize(te_nd, tr_nd);

@@ -595,7 +643,20 @@ void DiffusionIntegrator::AssembleElementMatrix2(
      Mult(dshape, invdfdx, dshapedxt);
      Mult(te_dshape, invdfdx, te_dshapedxt);
      // invdfdx, dshape, and te_dshape no longer needed
-      if (!MQ)
+      if (MQ)
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(te_dshapedxt, invdfdx, te_dshape);
+         AddMultABt(te_dshape, dshapedxt, elmat);
+      }
+      else if (VQ)
+      {
+         VQ->Eval(D, Trans, ip);
+         D *= w;
+         AddMultADAt(dshapedxt, D, elmat);
+      }
+      else
      {
         if (Q)
         {
@@ -604,13 +665,6 @@ void DiffusionIntegrator::AssembleElementMatrix2(
         dshapedxt *= w;
         AddMultABt(te_dshapedxt, dshapedxt, elmat);
      }
-      else
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(te_dshapedxt, invdfdx, te_dshape);
-         AddMultABt(te_dshape, dshapedxt, elmat);
-      }
   }
 }

@@ -622,12 +676,19 @@ void DiffusionIntegrator::AssembleElementVector(
   int dim = el.GetDim();
   double w;

+   if (VQ)
+   {
+      MFEM_VERIFY(VQ->GetVDim() == dim, "Unexpected dimension for VectorCoefficient");
+   }
+
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), invdfdx(dim), mq(dim);
+   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd,dim);
   invdfdx.SetSize(dim);
   mq.SetSize(dim);
+   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   vec.SetSize(dim);
   pointflux.SetSize(dim);
@@ -646,7 +707,7 @@ void DiffusionIntegrator::AssembleElementVector(
      CalcAdjugate(Tr.Jacobian(), invdfdx); // invdfdx = adj(J)
      w = ip.weight / Tr.Weight();

-      if (!MQ)
+      if (!MQ && !VQ)
      {
         dshape.MultTranspose(elfun, vec);
         invdfdx.MultTranspose(vec, pointflux);
@@ -657,11 +718,21 @@ void DiffusionIntegrator::AssembleElementVector(
      }
      else
      {
-
         dshape.MultTranspose(elfun, pointflux);
         invdfdx.MultTranspose(pointflux, vec);
-         MQ->Eval(mq, Tr, ip);
-         mq.Mult(vec, pointflux);
+         if (MQ)
+         {
+            MQ->Eval(mq, Tr, ip);
+            mq.Mult(vec, pointflux);
+         }
+         else
+         {
+            VQ->Eval(D, Tr, ip);
+            for (int j=0; j<dim; ++j)
+            {
+               pointflux[j] *= D[j];
+            }
+         }
      }
      pointflux *= w;
      invdfdx.Mult(pointflux, vec);
@@ -2194,16 +2265,7 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      // integrand is rational function if det(J) is not constant
-      int order = 2 * Trans.OrderGrad(&el); // order of the numerator
-      if (el.Space() == FunctionSpace::rQk)
-      {
-         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
-      }
-      else
-      {
-         ir = &IntRules.Get(el.GetGeomType(), order);
-      }
+      ir = &DiffusionIntegrator::GetRule(el,el);
   }

   elmat = 0.0;
@@ -2254,11 +2316,7 @@ void VectorDiffusionIntegrator::AssembleElementVector(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      // integrand is rational function if det(J) is not constant
-      int order = 2 * Tr.OrderGrad(&el); // order of the numerator
-      ir = (el.Space() == FunctionSpace::rQk) ?
-           &RefinedIntRules.Get(el.GetGeomType(), order) :
-           &IntRules.Get(el.GetGeomType(), order);
+      ir = &DiffusionIntegrator::GetRule(el,el);
   }

   elvect = 0.0;
@@ -95,6 +95,30 @@ public:
   //                         const FiniteElementSpace &test_fes,
   //                         Vector &emat);

+   /// Method defining matrix-free assembly.
+   /** The result of fully matrix-free assembly is stored internally so that it
+       can be used later in the methods AddMultMF() and AddMultTransposeMF(). */
+   virtual void AssembleMF(const FiniteElementSpace &fes);
+
+   /** Perform the action of integrator on the input @a x and add the result to
+       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
+       the element-wise discontinuous version of the FE space.
+
+       This method can be called only after the method AssembleMF() has been
+       called. */
+   virtual void AddMultMF(const Vector &x, Vector &y) const;
+
+   /** Perform the transpose action of integrator on the input @a x and add the
+       result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
+       represent the element-wise discontinuous version of the FE space.
+
+       This method can be called only after the method AssemblePA() has been
+       called. */
+   virtual void AddMultTransposeMF(const Vector &x, Vector &y) const;
+
+   /// Assemble diagonal and add it to Vector @a diag.
+   virtual void AssembleDiagonalMF(Vector &diag);
+
   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                        Vector &ea_data_int,
                                        Vector &ea_data_ext,
@@ -131,11 +155,22 @@ public:
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);

-   /// Perform the local action of the BilinearFormIntegrator
+   /// @brief Perform the local action of the BilinearFormIntegrator.
+   /// Note that the default implementation in the base class is general but not
+   /// efficient.
   virtual void AssembleElementVector(const FiniteElement &el,
                                      ElementTransformation &Tr,
                                      const Vector &elfun, Vector &elvect);

+   /// @brief Perform the local action of the BilinearFormIntegrator resulting
+   /// from a face integral term.
+   /// Note that the default implementation in the base class is general but not
+   /// efficient.
+   virtual void AssembleFaceVector(const FiniteElement &el1,
+                                   const FiniteElement &el2,
+                                   FaceElementTransformations &Tr,
+                                   const Vector &elfun, Vector &elvect);
+
   virtual void AssembleElementGrad(const FiniteElement &el,
                                    ElementTransformation &Tr,
                                    const Vector &elfun, DenseMatrix &elmat)
@@ -1863,6 +1898,7 @@ class DiffusionIntegrator: public BilinearFormIntegrator
 {
 protected:
   Coefficient *Q;
+   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;

 private:
@@ -1870,6 +1906,7 @@ private:
 #ifndef MFEM_THREAD_SAFE
   DenseMatrix dshape, dshapedxt, invdfdx, mq;
   DenseMatrix te_dshape, te_dshapedxt;
+   Vector D;
 #endif

   // PA extension
@@ -1878,54 +1915,30 @@ private:
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, dofs1D, quad1D;
   Vector pa_data;
-
-#ifdef MFEM_USE_CEED
+   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
   // CEED extension
   CeedData* ceedDataPtr;
-#endif

 public:
   /// Construct a diffusion integrator with coefficient Q = 1
   DiffusionIntegrator()
-   {
-      Q = NULL;
-      MQ = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : Q(NULL), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }

   /// Construct a diffusion integrator with a scalar coefficient q
   DiffusionIntegrator(Coefficient &q)
-      : Q(&q)
-   {
-      MQ = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : Q(&q), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+
+   /// Construct a diffusion integrator with a vector coefficient q
+   DiffusionIntegrator(VectorCoefficient &q)
+      : Q(NULL), VQ(&q), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }

   /// Construct a diffusion integrator with a matrix coefficient q
   DiffusionIntegrator(MatrixCoefficient &q)
-      : MQ(&q)
-   {
-      Q = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : Q(NULL), VQ(NULL), MQ(&q), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }

   virtual ~DiffusionIntegrator()
   {
-#ifdef MFEM_USE_CEED
      delete ceedDataPtr;
-#endif
   }

   /** Given a particular Finite Element computes the element stiffness matrix
@@ -1956,6 +1969,8 @@ public:

   using BilinearFormIntegrator::AssemblePA;

+   virtual void AssembleMF(const FiniteElementSpace &fes);
+
   virtual void AssemblePA(const FiniteElementSpace &fes);

   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
@@ -1963,12 +1978,14 @@ public:

   virtual void AssembleDiagonalPA(Vector &diag);

+   virtual void AssembleDiagonalMF(Vector &diag);
+
+   virtual void AddMultMF(const Vector&, Vector&) const;
+
   virtual void AddMultPA(const Vector&, Vector&) const;

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe);
-
-   void SetupPA(const FiniteElementSpace &fes);
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1986,39 +2003,22 @@ protected:
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;

-#ifdef MFEM_USE_CEED
   // CEED extension
   CeedData* ceedDataPtr;
-#endif

 public:
   MassIntegrator(const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir)
-   {
-      Q = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : BilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL),
+        ceedDataPtr(NULL) { }

   /// Construct a mass integrator with coefficient q
   MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(&q)
-   {
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : BilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL),
+        ceedDataPtr(NULL) { }

   virtual ~MassIntegrator()
   {
-#ifdef MFEM_USE_CEED
      delete ceedDataPtr;
-#endif
   }
   /** Given a particular Finite Element computes the element mass matrix
       elmat. */
@@ -2032,6 +2032,8 @@ public:

   using BilinearFormIntegrator::AssemblePA;

+   virtual void AssembleMF(const FiniteElementSpace &fes);
+
   virtual void AssemblePA(const FiniteElementSpace &fes);

   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
@@ -2039,6 +2041,10 @@ public:

   virtual void AssembleDiagonalPA(Vector &diag);

+   virtual void AssembleDiagonalMF(Vector &diag);
+
+   virtual void AddMultMF(const Vector&, Vector&) const;
+
   virtual void AddMultPA(const Vector&, Vector&) const;

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
@@ -2145,24 +2151,34 @@ protected:
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;

+   // CEED extension
+   CeedData* ceedDataPtr;
+
 public:
   /// Construct an integrator with coefficient 1.0
   VectorMassIntegrator()
-      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
+      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL), ceedDataPtr(NULL) { }
   /** Construct an integrator with scalar coefficient q.  If possible, save
       memory by using a scalar integrator since the resulting matrix is block
       diagonal with the same diagonal block repeated. */
   VectorMassIntegrator(Coefficient &q, int qo = 0)
-      : vdim(-1), Q(&q) { VQ = NULL; MQ = NULL; Q_order = qo; }
+      : vdim(-1), Q_order(qo), Q(&q), VQ(NULL), MQ(NULL), ceedDataPtr(NULL) { }
   VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir)
-      : BilinearFormIntegrator(ir), vdim(-1), Q(&q)
-   { VQ = NULL; MQ = NULL; Q_order = 0; }
+      : BilinearFormIntegrator(ir), vdim(-1), Q_order(0), Q(&q), VQ(NULL),
+        MQ(NULL), ceedDataPtr(NULL) { }
   /// Construct an integrator with diagonal coefficient q
   VectorMassIntegrator(VectorCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), VQ(&q) { Q = NULL; MQ = NULL; Q_order = qo; }
+      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(&q), MQ(NULL),
+        ceedDataPtr(NULL) { }
   /// Construct an integrator with matrix coefficient q
   VectorMassIntegrator(MatrixCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), MQ(&q) { Q = NULL; VQ = NULL; Q_order = qo; }
+      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(NULL), MQ(&q),
+        ceedDataPtr(NULL) { }
+
+   virtual ~VectorMassIntegrator()
+   {
+      delete ceedDataPtr;
+   }

   int GetVDim() const { return vdim; }
   void SetVDim(int vdim) { this->vdim = vdim; }
@@ -2176,8 +2192,11 @@ public:
                                       DenseMatrix &elmat);
   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &fes);
+   virtual void AssembleMF(const FiniteElementSpace &fes);
   virtual void AssembleDiagonalPA(Vector &diag);
+   virtual void AssembleDiagonalMF(Vector &diag);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultMF(const Vector &x, Vector &y) const;
 };


@@ -2541,13 +2560,23 @@ protected:
   int dim, sdim, ne, dofs1D, quad1D;
   Vector pa_data;

+   // CEED extension
+   CeedData* ceedDataPtr;
+
 private:
   DenseMatrix dshape, dshapedxt, pelmat;
   DenseMatrix Jinv, gshape;

 public:
-   VectorDiffusionIntegrator() { Q = NULL; }
-   VectorDiffusionIntegrator(Coefficient &q) { Q = &q; }
+   VectorDiffusionIntegrator()
+      : Q(NULL), ceedDataPtr(NULL) { }
+   VectorDiffusionIntegrator(Coefficient &q)
+      : Q(&q), ceedDataPtr(NULL) { }
+
+   virtual ~VectorDiffusionIntegrator()
+   {
+      delete ceedDataPtr;
+   }

   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
@@ -2557,8 +2586,11 @@ public:
                                      const Vector &elfun, Vector &elvect);
   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &fes);
+   virtual void AssembleMF(const FiniteElementSpace &fes);
   virtual void AssembleDiagonalPA(Vector &diag);
+   virtual void AssembleDiagonalMF(Vector &diag);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultMF(const Vector &x, Vector &y) const;
 };

 /** Integrator for the linear elasticity form:
@@ -3029,6 +3061,17 @@ protected:
   VectorCoefficient *VQ;
 };

-}

+
+// PA Diffusion Assemble 2D kernel
+template<const int T_SDIM>
+void PADiffusionSetup2D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const Array<double> &w,
+                        const Vector &j,
+                        const Vector &c,
+                        Vector &d);
+
+}
 #endif
@@ -0,0 +1,70 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "libceed/diffusion.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+void DiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
+{
+#ifdef MFEM_USE_CEED
+   // Assuming the same element type
+   fespace = &fes;
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNE() == 0) { return; }
+   const FiniteElement &el = *fes.GetFE(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedMFDiffusionAssemble(fes, *ir, *ceedDataPtr);
+   }
+#endif
+   mfem_error("Error: DiffusionIntegrator::AssembleMF only implemented with libCEED");
+}
+
+void DiffusionIntegrator::AssembleDiagonalMF(Vector &diag)
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAssembleDiagonal(ceedDataPtr, diag);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: DiffusionIntegrator::AssembleDiagonalMF only implemented with libCEED");
+   }
+}
+
+void DiffusionIntegrator::AddMultMF(const Vector &x, Vector &y) const
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAddMult(ceedDataPtr, x, y);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: DiffusionIntegrator::AddMultMF only implemented with libCEED");
+   }
+}
+
+}
@@ -80,28 +80,24 @@ static void OccaPADiffusionSetup3D(const int D1D,
 }
 #endif // MFEM_USE_OCCA

-// PA Diffusion Assemble 2D kernel
-template<const int T_SDIM>
-static void PADiffusionSetup2D(const int Q1D,
-                               const int NE,
-                               const Array<double> &w,
-                               const Vector &j,
-                               const Vector &c,
-                               Vector &d);
 template<>
 void PADiffusionSetup2D<2>(const int Q1D,
+                           const int coeffDim,
                           const int NE,
                           const Array<double> &w,
                           const Vector &j,
                           const Vector &c,
                           Vector &d)
 {
+   const bool symmetric = (coeffDim != 4);
   const bool const_c = c.Size() == 1;
+   MFEM_VERIFY(coeffDim < 3 ||
+               !const_c, "Constant matrix coefficient not supported");
   const auto W = Reshape(w.Read(), Q1D,Q1D);
   const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
+                  Reshape(c.Read(), coeffDim,Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D, symmetric ? 3 : 4, NE);
   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
   {
      MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -112,11 +108,39 @@ void PADiffusionSetup2D<2>(const int Q1D,
            const double J21 = J(qx,qy,1,0,e);
            const double J12 = J(qx,qy,0,1,e);
            const double J22 = J(qx,qy,1,1,e);
-            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-            const double c_detJ = W(qx,qy) * coeff / ((J11*J22)-(J21*J12));
-            D(qx,qy,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
-            D(qx,qy,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
-            D(qx,qy,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
+            const double w_detJ = W(qx,qy) / ((J11*J22)-(J21*J12));
+            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
+            {
+               // First compute entries of R = MJ^{-T}, without det J factor.
+               const double M11 = C(0,qx,qy,e);
+               const double M12 = C(1,qx,qy,e);
+               const double M21 = symmetric ? M12 : C(2,qx,qy,e);
+               const double M22 = symmetric ? C(2,qx,qy,e) : C(3,qx,qy,e);
+               const double R11 = M11*J22 - M12*J12;
+               const double R21 = M21*J22 - M22*J12;
+               const double R12 = -M11*J21 + M12*J11;
+               const double R22 = -M21*J21 + M22*J11;
+
+               // Now set y to J^{-1}R.
+               D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+               D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+               D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
+               (J22*R12 - J12*R22)); // 2,2 or 1,2
+               if (!symmetric)
+               {
+                  D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+               }
+            }
+            else // Vector or scalar coefficient
+            {
+               const double C1 = const_c ? C(0,0,0,0) : C(0,qx,qy,e);
+               const double C2 = const_c ? C(0,0,0,0) :
+                                 (coeffDim == 2 ? C(1,qx,qy,e) : C(0,qx,qy,e));
+
+               D(qx,qy,0,e) =  w_detJ * (C2*J12*J12 + C1*J22*J22); // 1,1
+               D(qx,qy,1,e) = -w_detJ * (C2*J12*J11 + C1*J22*J21); // 1,2
+               D(qx,qy,2,e) =  w_detJ * (C2*J11*J11 + C1*J21*J21); // 2,2
+            }
         }
      }
   });
@@ -125,12 +149,14 @@ void PADiffusionSetup2D<2>(const int Q1D,
 // PA Diffusion Assemble 2D kernel with 3D node coords
 template<>
 void PADiffusionSetup2D<3>(const int Q1D,
+                           const int coeffDim,
                           const int NE,
                           const Array<double> &w,
                           const Vector &j,
                           const Vector &c,
                           Vector &d)
 {
+   MFEM_VERIFY(coeffDim == 1, "Matrix and vector coefficients not supported");
   constexpr int DIM = 2;
   constexpr int SDIM = 3;
   const bool const_c = c.Size() == 1;
@@ -167,19 +193,23 @@ void PADiffusionSetup2D<3>(const int Q1D,
 }

 // PA Diffusion Assemble 3D kernel
-static void PADiffusionSetup3D(const int Q1D,
-                               const int NE,
-                               const Array<double> &w,
-                               const Vector &j,
-                               const Vector &c,
-                               Vector &d)
+void PADiffusionSetup3D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const Array<double> &w,
+                        const Vector &j,
+                        const Vector &c,
+                        Vector &d)
 {
+   const bool symmetric = (coeffDim != 9);
   const bool const_c = c.Size() == 1;
+   MFEM_VERIFY(coeffDim < 6 ||
+               !const_c, "Constant matrix coefficient not supported");
   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, 6, NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1,1) :
+                  Reshape(c.Read(), coeffDim,Q1D,Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, symmetric ? 6 : 9, NE);
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
      MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -200,8 +230,7 @@ static void PADiffusionSetup3D(const int Q1D,
               const double detJ = J11 * (J22 * J33 - J32 * J23) -
               /* */               J21 * (J12 * J33 - J32 * J13) +
               /* */               J31 * (J12 * J23 - J22 * J13);
-               const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-               const double c_detJ = W(qx,qy,qz) * coeff / detJ;
+               const double w_detJ = W(qx,qy,qz) / detJ;
               // adj(J)
               const double A11 = (J22 * J33) - (J23 * J32);
               const double A12 = (J32 * J13) - (J12 * J33);
@@ -212,13 +241,69 @@ static void PADiffusionSetup3D(const int Q1D,
               const double A31 = (J21 * J32) - (J31 * J22);
               const double A32 = (J31 * J12) - (J11 * J32);
               const double A33 = (J11 * J22) - (J12 * J21);
-               // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-               D(qx,qy,qz,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-               D(qx,qy,qz,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-               D(qx,qy,qz,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-               D(qx,qy,qz,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-               D(qx,qy,qz,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-               D(qx,qy,qz,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
+
+               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+               {
+                  // Compute entries of R = MJ^{-T} = M adj(J)^T, without det J.
+                  const double M11 = C(0, qx,qy,qz, e);
+                  const double M12 = C(1, qx,qy,qz, e);
+                  const double M13 = C(2, qx,qy,qz, e);
+                  const double M21 = (!symmetric) ? C(3, qx,qy,qz, e) : M12;
+                  const double M22 = (!symmetric) ? C(4, qx,qy,qz, e) : C(3, qx,qy,qz, e);
+                  const double M23 = (!symmetric) ? C(5, qx,qy,qz, e) : C(4, qx,qy,qz, e);
+                  const double M31 = (!symmetric) ? C(6, qx,qy,qz, e) : M13;
+                  const double M32 = (!symmetric) ? C(7, qx,qy,qz, e) : M23;
+                  const double M33 = (!symmetric) ? C(8, qx,qy,qz, e) : C(5, qx,qy,qz, e);
+
+                  const double R11 = M11*A11 + M12*A12 + M13*A13;
+                  const double R12 = M11*A21 + M12*A22 + M13*A23;
+                  const double R13 = M11*A31 + M12*A32 + M13*A33;
+                  const double R21 = M21*A11 + M22*A12 + M23*A13;
+                  const double R22 = M21*A21 + M22*A22 + M23*A23;
+                  const double R23 = M21*A31 + M22*A32 + M23*A33;
+                  const double R31 = M31*A11 + M32*A12 + M33*A13;
+                  const double R32 = M31*A21 + M32*A22 + M33*A23;
+                  const double R33 = M31*A31 + M32*A32 + M33*A33;
+
+                  // Now set D to J^{-1} R = adj(J) R
+                  D(qx,qy,qz,0,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+                  const double D12 = w_detJ * (A11*R12 + A12*R22 + A13*R32);
+                  D(qx,qy,qz,1,e) = D12; // 1,2
+                  D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+
+                  const double D21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
+                  const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
+                  const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
+
+                  const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
+
+                  D(qx,qy,qz,3,e) = symmetric ? D22 : D21; // 2,2 or 2,1
+                  D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
+                  D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3
+
+                  if (!symmetric)
+                  {
+                     D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+                     D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+                     D(qx,qy,qz,8,e) = D33; // 3,3
+                  }
+               }
+               else  // Vector or scalar coefficient version
+               {
+                  const double C1 = const_c ? C(0,0,0,0,0) : C(0,qx,qy,qz,e);
+                  const double C2 = const_c ? C(0,0,0,0,0) :
+                                    (coeffDim == 3 ? C(1,qx,qy,qz,e) : C(0,qx,qy,qz,e));
+                  const double C3 = const_c ? C(0,0,0,0,0) :
+                                    (coeffDim == 3 ? C(2,qx,qy,qz,e) : C(0,qx,qy,qz,e));
+
+                  // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+                  D(qx,qy,qz,0,e) = w_detJ * (C1*A11*A11 + C2*A12*A12 + C3*A13*A13); // 1,1
+                  D(qx,qy,qz,1,e) = w_detJ * (C1*A11*A21 + C2*A12*A22 + C3*A13*A23); // 2,1
+                  D(qx,qy,qz,2,e) = w_detJ * (C1*A11*A31 + C2*A12*A32 + C3*A13*A33); // 3,1
+                  D(qx,qy,qz,3,e) = w_detJ * (C1*A21*A21 + C2*A22*A22 + C3*A23*A23); // 2,2
+                  D(qx,qy,qz,4,e) = w_detJ * (C1*A21*A31 + C2*A22*A32 + C3*A23*A33); // 3,2
+                  D(qx,qy,qz,5,e) = w_detJ * (C1*A31*A31 + C2*A32*A32 + C3*A33*A33); // 3,3
+               }
            }
         }
      }
@@ -229,6 +314,7 @@ static void PADiffusionSetup(const int dim,
                             const int sdim,
                             const int D1D,
                             const int Q1D,
+                             const int coeffDim,
                             const int NE,
                             const Array<double> &W,
                             const Vector &J,
@@ -247,8 +333,8 @@ static void PADiffusionSetup(const int dim,
 #else
      MFEM_CONTRACT_VAR(D1D);
 #endif // MFEM_USE_OCCA
-      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, NE, W, J, C, D); }
-      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, NE, W, J, C, D); }
+      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, coeffDim, NE, W, J, C, D); }
+      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, coeffDim, NE, W, J, C, D); }
   }
   if (dim == 3)
   {
@@ -259,11 +345,11 @@ static void PADiffusionSetup(const int dim,
         return;
      }
 #endif // MFEM_USE_OCCA
-      PADiffusionSetup3D(Q1D, NE, W, J, C, D);
+      PADiffusionSetup3D(Q1D, coeffDim, NE, W, J, C, D);
   }
 }

-void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
+void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
   // Assuming the same element type
   fespace = &fes;
@@ -271,16 +357,13 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
   if (mesh->GetNE() == 0) { return; }
   const FiniteElement &el = *fes.GetFE(0);
   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
-#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      if (ceedDataPtr) { delete ceedDataPtr; }
-      CeedData* ptr = new CeedData();
-      ceedDataPtr = ptr;
-      InitCeedCoeff(Q, ptr);
-      return CeedPADiffusionAssemble(fes, *ir, *ptr);
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedPADiffusionAssemble(fes, *ir, *ceedDataPtr);
   }
-#endif
   const int dims = el.GetDim();
   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
   const int nq = ir->GetNPoints();
@@ -291,9 +374,80 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, Device::GetDeviceMemoryType());
+   int coeffDim = 1;
   Vector coeff;
-   if (Q == nullptr)
+   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
+   if (MQ)
+   {
+      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+      const int MQsymmDim = MQ->GetWidth() * (MQ->GetWidth() + 1) / 2;
+
+      const int MQdim = MQ->IsSymmetric() ? MQsymmDim : MQfullDim;
+      coeffDim = MQdim;
+
+      coeff.SetSize(MQdim * nq * ne);
+      symmetric = MQ ? MQ->IsSymmetric() : true;
+
+      DenseMatrix M;
+      Vector Msymm;
+      if (symmetric)
+      {
+         Msymm.SetSize(MQsymmDim);
+      }
+      else
+      {
+         M.SetSize(dim);
+      }
+
+      auto C = Reshape(coeff.HostWrite(), MQdim, nq, ne);
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            if (MQ->IsSymmetric())
+            {
+               MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<MQsymmDim; ++i)
+               {
+                  C(i, p, e) = Msymm[i];
+               }
+            }
+            else
+            {
+               MQ->Eval(M, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<dim; ++i)
+                  for (int j=0; j<dim; ++j)
+                  {
+                     C(j+(i*dim), p, e) = M(i,j);
+                  }
+            }
+         }
+      }
+   }
+   else if (VQ)
+   {
+      MFEM_VERIFY(VQ->GetVDim() == dim, "");
+      coeffDim = VQ->GetVDim();
+      coeff.SetSize(coeffDim * nq * ne);
+      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+      Vector D(coeffDim);
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            VQ->Eval(D, *tr, ir->IntPoint(p));
+            for (int i=0; i<coeffDim; ++i)
+            {
+               C(i, p, e) = D[i];
+            }
+         }
+      }
+   }
+   else if (Q == nullptr)
   {
      coeff.SetSize(1);
      coeff(0) = 1.0;
@@ -329,18 +483,15 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
         }
      }
   }
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
-                    coeff, pa_data);
+   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
+                   Device::GetDeviceMemoryType());
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
+                    geom->J, coeff, pa_data);
 }

-void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   SetupPA(fes);
-}
-
-
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionDiagonal2D(const int NE,
+                                  const bool symmetric,
                                  const Array<double> &b,
                                  const Array<double> &g,
                                  const Vector &d,
@@ -354,9 +505,9 @@ static void PADiffusionDiagonal2D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
-   // note the different shape for D, this is a (symmetric) matrix so we only
+   // note the different shape for D, if this is a symmetric matrix we only
   // store necessary entries
-   auto D = Reshape(d.Read(), Q1D*Q1D, 3, NE);
+   auto D = Reshape(d.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL(e, NE,
   {
@@ -378,12 +529,13 @@ static void PADiffusionDiagonal2D(const int NE,
            for (int qy = 0; qy < Q1D; ++qy)
            {
               const int q = qx + qy * Q1D;
-               const double D0 = D(q,0,e);
-               const double D1 = D(q,1,e);
-               const double D2 = D(q,2,e);
-               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D0;
-               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * D1;
-               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D2;
+               const double D00 = D(q,0,e);
+               const double D10 = D(q,1,e);
+               const double D01 = symmetric ? D10 : D(q,2,e);
+               const double D11 = symmetric ? D(q,2,e) : D(q,3,e);
+               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D00;
+               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * (D01 + D10);
+               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D11;
            }
         }
      }
@@ -395,7 +547,6 @@ static void PADiffusionDiagonal2D(const int NE,
            {
               Y(dx,dy,e) += G(qx, dx) * G(qx, dx) * QD0[qx][dy];
               Y(dx,dy,e) += G(qx, dx) * B(qx, dx) * QD1[qx][dy];
-               Y(dx,dy,e) += B(qx, dx) * G(qx, dx) * QD1[qx][dy];
               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD2[qx][dy];
            }
         }
@@ -406,6 +557,7 @@ static void PADiffusionDiagonal2D(const int NE,
 // Shared memory PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionDiagonal2D(const int NE,
+                                      const bool symmetric,
                                      const Array<double> &b_,
                                      const Array<double> &g_,
                                      const Vector &d_,
@@ -422,7 +574,7 @@ static void SmemPADiffusionDiagonal2D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
   {
@@ -435,10 +587,10 @@ static void SmemPADiffusionDiagonal2D(const int NE,
      MFEM_SHARED double BG[2][MQ1*MD1];
      double (*B)[MD1] = (double (*)[MD1]) (BG+0);
      double (*G)[MD1] = (double (*)[MD1]) (BG+1);
-      MFEM_SHARED double QD[4][NBZ][MD1][MQ1];
+      MFEM_SHARED double QD[3][NBZ][MD1][MQ1];
      double (*QD0)[MD1] = (double (*)[MD1])(QD[0] + tidz);
      double (*QD1)[MD1] = (double (*)[MD1])(QD[1] + tidz);
-      double (*QD2)[MD1] = (double (*)[MD1])(QD[3] + tidz);
+      double (*QD2)[MD1] = (double (*)[MD1])(QD[2] + tidz);
      if (tidz == 0)
      {
         MFEM_FOREACH_THREAD(d,y,D1D)
@@ -461,17 +613,18 @@ static void SmemPADiffusionDiagonal2D(const int NE,
            for (int qy = 0; qy < Q1D; ++qy)
            {
               const int q = qx + qy * Q1D;
-               const double D0 = D(q,0,e);
-               const double D1 = D(q,1,e);
-               const double D2 = D(q,2,e);
+               const double D00 = D(q,0,e);
+               const double D10 = D(q,1,e);
+               const double D01 = symmetric ? D10 : D(q,2,e);
+               const double D11 = symmetric ? D(q,2,e) : D(q,3,e);
               const double By = B[qy][dy];
               const double Gy = G[qy][dy];
               const double BB = By * By;
               const double BG = By * Gy;
               const double GG = Gy * Gy;
-               QD0[qx][dy] += BB * D0;
-               QD1[qx][dy] += BG * D1;
-               QD2[qx][dy] += GG * D2;
+               QD0[qx][dy] += BB * D00;
+               QD1[qx][dy] += BG * (D01 + D10);
+               QD2[qx][dy] += GG * D11;
            }
         }
      }
@@ -489,7 +642,6 @@ static void SmemPADiffusionDiagonal2D(const int NE,
               const double GG = Gx * Gx;
               Y(dx,dy,e) += GG * QD0[qx][dy];
               Y(dx,dy,e) += BG * QD1[qx][dy];
-               Y(dx,dy,e) += BG * QD1[qx][dy];
               Y(dx,dy,e) += BB * QD2[qx][dy];
            }
         }
@@ -499,6 +651,7 @@ static void SmemPADiffusionDiagonal2D(const int NE,

 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionDiagonal3D(const int NE,
+                                  const bool symmetric,
                                  const Array<double> &b,
                                  const Array<double> &g,
                                  const Vector &d,
@@ -515,7 +668,7 @@ static void PADiffusionDiagonal3D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL(e, NE,
   {
@@ -540,9 +693,10 @@ static void PADiffusionDiagonal3D(const int NE,
                     for (int qz = 0; qz < Q1D; ++qz)
                     {
                        const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int k = j >= i ?
+                        const int ksym = j >= i ?
                        3 - (3-i)*(2-i)/2 + j:
                        3 - (3-j)*(2-j)/2 + i;
+                        const int k = symmetric ? ksym : (i*DIM) + j;
                        const double O = Q(q,k,e);
                        const double Bz = B(qz,dz);
                        const double Gz = G(qz,dz);
@@ -598,6 +752,7 @@ static void PADiffusionDiagonal3D(const int NE,
 // Shared memory PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionDiagonal3D(const int NE,
+                                      const bool symmetric,
                                      const Array<double> &b_,
                                      const Array<double> &g_,
                                      const Vector &d_,
@@ -614,7 +769,7 @@ static void SmemPADiffusionDiagonal3D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
@@ -655,9 +810,10 @@ static void SmemPADiffusionDiagonal3D(const int NE,
                     for (int qz = 0; qz < Q1D; ++qz)
                     {
                        const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int k = j >= i ?
-                                      3 - (3-i)*(2-i)/2 + j:
-                                      3 - (3-j)*(2-j)/2 + i;
+                        const int ksym = j >= i ?
+                                         3 - (3-i)*(2-i)/2 + j:
+                                         3 - (3-j)*(2-j)/2 + i;
+                        const int k = symmetric ? ksym : (i*DIM) + j;
                        const double O = D(q,k,e);
                        const double Bz = B[qz][dz];
                        const double Gz = G[qz][dz];
@@ -716,6 +872,7 @@ static void PADiffusionAssembleDiagonal(const int dim,
                                        const int D1D,
                                        const int Q1D,
                                        const int NE,
+                                        const bool symm,
                                        const Array<double> &B,
                                        const Array<double> &G,
                                        const Vector &D,
@@ -725,30 +882,30 @@ static void PADiffusionAssembleDiagonal(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
-         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,B,G,D,Y);
-         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,B,G,D,Y);
-         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,B,G,D,Y);
-         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,B,G,D,Y);
-         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,B,G,D,Y);
-         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,B,G,D,Y);
-         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,B,G,D,Y);
-         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,B,G,D,Y);
-         default: return PADiffusionDiagonal2D(NE,B,G,D,Y,D1D,Q1D);
+         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,symm,B,G,D,Y);
+         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,symm,B,G,D,Y);
+         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,symm,B,G,D,Y);
+         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,symm,B,G,D,Y);
+         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,symm,B,G,D,Y);
+         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,symm,B,G,D,Y);
+         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,symm,B,G,D,Y);
+         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,symm,B,G,D,Y);
+         default: return PADiffusionDiagonal2D(NE,symm,B,G,D,Y,D1D,Q1D);
      }
   }
   else if (dim == 3)
   {
      switch ((D1D << 4 ) | Q1D)
      {
-         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,B,G,D,Y);
-         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,B,G,D,Y);
-         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,B,G,D,Y);
-         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,B,G,D,Y);
-         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,B,G,D,Y);
-         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,B,G,D,Y);
-         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,B,G,D,Y);
-         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,B,G,D,Y);
-         default: return PADiffusionDiagonal3D(NE,B,G,D,Y,D1D,Q1D);
+         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,symm,B,G,D,Y);
+         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,symm,B,G,D,Y);
+         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,symm,B,G,D,Y);
+         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,symm,B,G,D,Y);
+         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,symm,B,G,D,Y);
+         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,symm,B,G,D,Y);
+         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,symm,B,G,D,Y);
+         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,symm,B,G,D,Y);
+         default: return PADiffusionDiagonal3D(NE,symm,B,G,D,Y,D1D,Q1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -756,15 +913,14 @@ static void PADiffusionAssembleDiagonal(const int dim,

 void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAssembleDiagonalPA(ceedDataPtr, diag);
+      CeedAssembleDiagonal(ceedDataPtr, diag);
   }
   else
-#endif
   {
-      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
+      if (pa_data.Size()==0) { AssemblePA(*fespace); }
+      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
                                  maps->B, maps->G, pa_data, diag);
   }
 }
@@ -873,6 +1029,7 @@ static void OccaPADiffusionApply3D(const int D1D,
 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionApply2D(const int NE,
+                               const bool symmetric,
                               const Array<double> &b_,
                               const Array<double> &g_,
                               const Array<double> &bt_,
@@ -891,7 +1048,7 @@ static void PADiffusionApply2D(const int NE,
   auto G = Reshape(g_.Read(), Q1D, D1D);
   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
   auto Gt = Reshape(gt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
   auto X = Reshape(x_.Read(), D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL(e, NE,
@@ -947,14 +1104,15 @@ static void PADiffusionApply2D(const int NE,
            const int q = qx + qy * Q1D;

            const double O11 = D(q,0,e);
-            const double O12 = D(q,1,e);
-            const double O22 = D(q,2,e);
+            const double O21 = D(q,1,e);
+            const double O12 = symmetric ? O21 : D(q,2,e);
+            const double O22 = symmetric ? D(q,2,e) : D(q,3,e);

            const double gradX = grad[qy][qx][0];
            const double gradY = grad[qy][qx][1];

            grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-            grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+            grad[qy][qx][1] = (O21 * gradX) + (O22 * gradY);
         }
      }
      for (int qy = 0; qy < Q1D; ++qy)
@@ -993,6 +1151,7 @@ static void PADiffusionApply2D(const int NE,
 // Shared memory PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionApply2D(const int NE,
+                                   const bool symmetric,
                                   const Array<double> &b_,
                                   const Array<double> &g_,
                                   const Vector &d_,
@@ -1010,7 +1169,7 @@ static void SmemPADiffusionApply2D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
@@ -1092,12 +1251,13 @@ static void SmemPADiffusionApply2D(const int NE,
         {
            const int q = (qx + ((qy) * Q1D));
            const double O11 = D(q,0,e);
-            const double O12 = D(q,1,e);
-            const double O22 = D(q,2,e);
+            const double O21 = D(q,1,e);
+            const double O12 = symmetric ? O21 : D(q,2,e);
+            const double O22 = symmetric ? D(q,2,e) : D(q,3,e);
            const double gX = QQ0[qy][qx];
            const double gY = QQ1[qy][qx];
            QQ0[qy][qx] = (O11 * gX) + (O12 * gY);
-            QQ1[qy][qx] = (O12 * gX) + (O22 * gY);
+            QQ1[qy][qx] = (O21 * gX) + (O22 * gY);
         }
      }
      MFEM_SYNC_THREAD;
@@ -1149,6 +1309,7 @@ static void SmemPADiffusionApply2D(const int NE,
 // PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionApply3D(const int NE,
+                               const bool symmetric,
                               const Array<double> &b,
                               const Array<double> &g,
                               const Array<double> &bt,
@@ -1166,7 +1327,7 @@ static void PADiffusionApply3D(const int NE,
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto Bt = Reshape(bt.Read(), D1D, Q1D);
   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
   auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL(e, NE,
@@ -1257,15 +1418,18 @@ static void PADiffusionApply3D(const int NE,
               const double O11 = D(q,0,e);
               const double O12 = D(q,1,e);
               const double O13 = D(q,2,e);
-               const double O22 = D(q,3,e);
-               const double O23 = D(q,4,e);
-               const double O33 = D(q,5,e);
+               const double O21 = symmetric ? O12 : D(q,3,e);
+               const double O22 = symmetric ? D(q,3,e) : D(q,4,e);
+               const double O23 = symmetric ? D(q,4,e) : D(q,5,e);
+               const double O31 = symmetric ? O13 : D(q,6,e);
+               const double O32 = symmetric ? O23 : D(q,7,e);
+               const double O33 = symmetric ? D(q,5,e) : D(q,8,e);
               const double gradX = grad[qz][qy][qx][0];
               const double gradY = grad[qz][qy][qx][1];
               const double gradZ = grad[qz][qy][qx][2];
               grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ);
-               grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ);
-               grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ);
+               grad[qz][qy][qx][1] = (O21*gradX)+(O22*gradY)+(O23*gradZ);
+               grad[qz][qy][qx][2] = (O31*gradX)+(O32*gradY)+(O33*gradZ);
            }
         }
      }
@@ -1364,6 +1528,7 @@ static MFEM_HOST_DEVICE inline double sign(const int q, const int d)

 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
+                                   const bool symmetric,
                                   const Array<double> &b_,
                                   const Array<double> &g_,
                                   const Vector &d_,
@@ -1380,7 +1545,7 @@ static void SmemPADiffusionApply3D(const int NE,
   MFEM_VERIFY(Q1D <= M1Q, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
@@ -1527,15 +1692,18 @@ static void SmemPADiffusionApply3D(const int NE,
               const double O11 = d(qx,qy,qz,0,e);
               const double O12 = d(qx,qy,qz,1,e);
               const double O13 = d(qx,qy,qz,2,e);
-               const double O22 = d(qx,qy,qz,3,e);
-               const double O23 = d(qx,qy,qz,4,e);
-               const double O33 = d(qx,qy,qz,5,e);
+               const double O21 = symmetric ? O12 : d(qx,qy,qz,3,e);
+               const double O22 = symmetric ? d(qx,qy,qz,3,e) : d(qx,qy,qz,4,e);
+               const double O23 = symmetric ? d(qx,qy,qz,4,e) : d(qx,qy,qz,5,e);
+               const double O31 = symmetric ? O13 : d(qx,qy,qz,6,e);
+               const double O32 = symmetric ? O23 : d(qx,qy,qz,7,e);
+               const double O33 = symmetric ? d(qx,qy,qz,5,e) : d(qx,qy,qz,8,e);
               const double gX = u[qz];
               const double gY = v[qz];
               const double gZ = w[qz];
               QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
-               QQQ1[qz][qy][qx] = (O12*gX) + (O22*gY) + (O23*gZ);
-               QQQ2[qz][qy][qx] = (O13*gX) + (O23*gY) + (O33*gZ);
+               QQQ1[qz][qy][qx] = (O21*gX) + (O22*gY) + (O23*gZ);
+               QQQ2[qz][qy][qx] = (O31*gX) + (O32*gY) + (O33*gZ);
            }
         }
      }
@@ -1656,6 +1824,7 @@ static void PADiffusionApply(const int dim,
                             const int D1D,
                             const int Q1D,
                             const int NE,
+                             const bool symm,
                             const Array<double> &B,
                             const Array<double> &G,
                             const Array<double> &Bt,
@@ -1686,15 +1855,15 @@ static void PADiffusionApply(const int dim,
   {
      switch (ID)
      {
-         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
-         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
-         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,B,G,D,X,Y);
-         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,B,G,D,X,Y);
-         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,B,G,D,X,Y);
-         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,B,G,D,X,Y);
-         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,B,G,D,X,Y);
-         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,B,G,D,X,Y);
-         default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
+         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
+         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
+         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
+         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
+         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
+         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
+         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
+         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }

@@ -1702,16 +1871,16 @@ static void PADiffusionApply(const int dim,
   {
      switch (ID)
      {
-         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,D,X,Y);
-         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,D,X,Y);
-         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,B,G,D,X,Y);
-         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,B,G,D,X,Y);
-         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,B,G,D,X,Y);
-         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,B,G,D,X,Y);
-         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,B,G,D,X,Y);
-         default:   return PADiffusionApply3D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,symm,B,G,D,X,Y);
+         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,symm,B,G,D,X,Y);
+         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,symm,B,G,D,X,Y);
+         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,symm,B,G,D,X,Y);
+         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,symm,B,G,D,X,Y);
+         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,symm,B,G,D,X,Y);
+         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
+         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
+         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
+         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -1720,15 +1889,13 @@ static void PADiffusionApply(const int dim,
 // PA Diffusion Apply kernel
 void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAddMultPA(ceedDataPtr, x, y);
+      CeedAddMult(ceedDataPtr, x, y);
   }
   else
-#endif
   {
-      PADiffusionApply(dim, dofs1D, quad1D, ne,
+      PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,
                       maps->B, maps->G, maps->Bt, maps->Gt,
                       pa_data, x, y);
   }
@@ -19,174 +19,6 @@ using namespace std;
 namespace mfem
 {

-// PA H(curl) Mass Assemble 2D kernel
-void PAHcurlSetup2D(const int Q1D,
-                    const int coeffDim,
-                    const int NE,
-                    const Array<double> &w,
-                    const Vector &j,
-                    Vector &coeff,
-                    Vector &op)
-{
-   const int NQ = Q1D*Q1D;
-   const bool symmetric = (coeffDim != 4);
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto C = Reshape(coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), NQ, symmetric ? 3 : 4, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-
-         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
-         {
-            // First compute entries of R = MJ^{-T}, without det J factor.
-            const double M11 = C(0, q, e);
-            const double M12 = C(1, q, e);
-            const double M21 = symmetric ? M12 : C(2, q, e);
-            const double M22 = symmetric ? C(2, q, e) : C(3, q, e);
-            const double R11 = M11*J22 - M12*J12;
-            const double R21 = M21*J22 - M22*J12;
-            const double R12 = -M11*J21 + M12*J11;
-            const double R22 = -M21*J21 + M22*J11;
-
-            // Now set y to J^{-1}R.
-            const double w_detJ = W[q] / ((J11*J22)-(J21*J12));
-            y(q,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-            y(q,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-            y(q,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
-            (J22*R12 - J12*R22)); // 2,2 or 1,2
-            if (!symmetric)
-            {
-               y(q,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-            }
-         }
-         else  // Vector or scalar coefficient version
-         {
-            const double c_detJ1 = W[q] * C(0, q, e) / ((J11*J22)-(J21*J12));
-            const double c_detJ2 = (coeffDim == 2) ? W[q] * C(1, q, e)
-                                   / ((J11*J22)-(J21*J12)) : c_detJ1;
-            y(q,0,e) =  (c_detJ2*J12*J12 + c_detJ1*J22*J22); // 1,1
-            y(q,1,e) = -(c_detJ2*J12*J11 + c_detJ1*J22*J21); // 1,2
-            y(q,2,e) =  (c_detJ2*J11*J11 + c_detJ1*J21*J21); // 2,2
-         }
-      }
-   });
-}
-
-// PA H(curl) Mass Assemble 3D kernel
-void PAHcurlSetup3D(const int Q1D,
-                    const int coeffDim,
-                    const int NE,
-                    const Array<double> &w,
-                    const Vector &j,
-                    Vector &coeff,
-                    Vector &op)
-{
-   const int NQ = Q1D*Q1D*Q1D;
-   const bool symmetric = (coeffDim != 9);
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto C = Reshape(coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), NQ, symmetric ? 6 : 9, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double detJ = J11 * (J22 * J33 - J32 * J23) -
-         /* */               J21 * (J12 * J33 - J32 * J13) +
-         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double w_detJ = W[q] / detJ;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-
-         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-         {
-            // First compute entries of R = MJ^{-T} = M adj(J)^T, without det J factor.
-            const double M11 = C(0, q, e);
-            const double M12 = C(1, q, e);
-            const double M13 = C(2, q, e);
-            const double M21 = (!symmetric) ? C(3, q, e) : M12;
-            const double M22 = (!symmetric) ? C(4, q, e) : C(3, q, e);
-            const double M23 = (!symmetric) ? C(5, q, e) : C(4, q, e);
-            const double M31 = (!symmetric) ? C(6, q, e) : M13;
-            const double M32 = (!symmetric) ? C(7, q, e) : M23;
-            const double M33 = (!symmetric) ? C(8, q, e) : C(5, q, e);
-
-            const double R11 = M11*A11 + M12*A12 + M13*A13;
-            const double R12 = M11*A21 + M12*A22 + M13*A23;
-            const double R13 = M11*A31 + M12*A32 + M13*A33;
-            const double R21 = M21*A11 + M22*A12 + M23*A13;
-            const double R22 = M21*A21 + M22*A22 + M23*A23;
-            const double R23 = M21*A31 + M22*A32 + M23*A33;
-            const double R31 = M31*A11 + M32*A12 + M33*A13;
-            const double R32 = M31*A21 + M32*A22 + M33*A23;
-            const double R33 = M31*A31 + M32*A32 + M33*A33;
-
-            // Now set y to J^{-1} R = adj(J) R
-            y(q,0,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-            const double Y12 = w_detJ * (A11*R12 + A12*R22 + A13*R32);
-            y(q,1,e) = Y12; // 1,2
-            y(q,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-
-            const double Y21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
-            const double Y22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
-            const double Y23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
-
-            const double Y33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
-
-            y(q,3,e) = symmetric ? Y22 : Y21; // 2,2 or 2,1
-            y(q,4,e) = symmetric ? Y23 : Y22; // 2,3 or 2,2
-            y(q,5,e) = symmetric ? Y33 : Y23; // 3,3 or 2,3
-
-            if (!symmetric)
-            {
-               y(q,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-               y(q,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-               y(q,8,e) = Y33; // 3,3
-            }
-         }
-         else  // Vector or scalar coefficient version
-         {
-            const double D1 = C(0, q, e);
-            const double D2 = coeffDim == 3 ? C(1, q, e) : D1;
-            const double D3 = coeffDim == 3 ? C(2, q, e) : D1;
-            // detJ J^{-1} D J^{-T} = (1/detJ) adj(J) D adj(J)^T
-            y(q,0,e) = w_detJ * (D1*A11*A11 + D2*A12*A12 + D3*A13*A13); // 1,1
-            y(q,1,e) = w_detJ * (D1*A11*A21 + D2*A12*A22 + D3*A13*A23); // 2,1
-            y(q,2,e) = w_detJ * (D1*A11*A31 + D2*A12*A32 + D3*A13*A33); // 3,1
-            y(q,3,e) = w_detJ * (D1*A21*A21 + D2*A22*A22 + D3*A23*A23); // 2,2
-            y(q,4,e) = w_detJ * (D1*A21*A31 + D2*A22*A32 + D3*A23*A33); // 3,2
-            y(q,5,e) = w_detJ * (D1*A31*A31 + D2*A32*A32 + D3*A33*A33); // 3,3
-         }
-      }
-   });
-}
-
 void PAHcurlMassApply2D(const int D1D,
                        const int Q1D,
                        const int NE,
@@ -0,0 +1,71 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "libceed/mass.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+void MassIntegrator::AssembleMF(const FiniteElementSpace &fes)
+{
+#ifdef MFEM_USE_CEED
+   // Assuming the same element type
+   fespace = &fes;
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNE() == 0) { return; }
+   const FiniteElement &el = *fes.GetFE(0);
+   ElementTransformation *T = mesh->GetElementTransformation(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedMFMassAssemble(fes, *ir, *ceedDataPtr);
+   }
+#endif
+   mfem_error("Error: MassIntegrator::AssembleMF only implemented with libCEED");
+}
+
+void MassIntegrator::AddMultMF(const Vector &x, Vector &y) const
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAddMult(ceedDataPtr, x, y);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: MassIntegrator::AddMultMF only implemented with libCEED");
+   }
+}
+
+void MassIntegrator::AssembleDiagonalMF(Vector &diag)
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAssembleDiagonal(ceedDataPtr, diag);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: MassIntegrator::AssembleDiagonalMF only implemented with libCEED");
+   }
+}
+
+} // namespace mfem
@@ -23,7 +23,7 @@ namespace mfem

 // PA Mass Assemble kernel

-void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
+void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
   // Assuming the same element type
   fespace = &fes;
@@ -32,16 +32,13 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
   const FiniteElement &el = *fes.GetFE(0);
   ElementTransformation *T = mesh->GetElementTransformation(0);
   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
-#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      if (ceedDataPtr) { delete ceedDataPtr; }
-      CeedData* ptr = new CeedData();
-      ceedDataPtr = ptr;
-      InitCeedCoeff(Q, ptr);
-      return CeedPAMassAssemble(fes, *ir, *ptr);
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedPAMassAssemble(fes, *ir, *ceedDataPtr);
   }
-#endif
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
@@ -155,12 +152,6 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
   }
 }

-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   SetupPA(fes);
-}
-
-
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PAMassAssembleDiagonal2D(const int NE,
                                     const Array<double> &b,
@@ -468,13 +459,11 @@ static void PAMassAssembleDiagonal(const int dim, const int D1D,

 void MassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAssembleDiagonalPA(ceedDataPtr, diag);
+      CeedAssembleDiagonal(ceedDataPtr, diag);
   }
   else
-#endif
   {
      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
   }
@@ -1227,13 +1216,11 @@ static void PAMassApply(const int dim,

 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAddMultPA(ceedDataPtr, x, y);
+      CeedAddMult(ceedDataPtr, x, y);
   }
   else
-#endif
   {
      PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
   }
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "libceed/diffusion.hpp"

 using namespace std;

@@ -129,6 +130,13 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   const FiniteElement &el = *fes.GetFE(0);
   const IntegrationRule *ir
      = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedPADiffusionAssemble(fes, *ir, * ceedDataPtr);
+   }
   const int dims = el.GetDim();
   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
   const int nq = ir->GetNPoints();
@@ -506,33 +514,40 @@ void PAVectorDiffusionApply3D(const int NE,
 // PA Diffusion Apply kernel
 void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const int D1D = dofs1D;
-   const int Q1D = quad1D;
-   const Array<double> &B = maps->B;
-   const Array<double> &G = maps->G;
-   const Array<double> &Bt = maps->Bt;
-   const Array<double> &Gt = maps->Gt;
-   const Vector &D = pa_data;
-
-   if (dim == 2 && sdim == 3)
+   if (DeviceCanUseCeed())
   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
-         default:
-            return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
-      }
+      CeedAddMult(ceedDataPtr, x, y);
   }
-   if (dim == 2 && sdim == 2)
-   { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
+   else
+   {
+      const int D1D = dofs1D;
+      const int Q1D = quad1D;
+      const Array<double> &B = maps->B;
+      const Array<double> &G = maps->G;
+      const Array<double> &Bt = maps->Bt;
+      const Array<double> &Gt = maps->Gt;
+      const Vector &D = pa_data;

-   if (dim == 3 && sdim == 3)
-   { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
+      if (dim == 2 && sdim == 3)
+      {
+         switch ((dofs1D << 4 ) | quad1D)
+         {
+            case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
+            default:
+               return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
+         }
+      }
+      if (dim == 2 && sdim == 2)
+      { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }

-   MFEM_ABORT("Unknown kernel.");
+      if (dim == 3 && sdim == 3)
+      { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
+
+      MFEM_ABORT("Unknown kernel.");
+   }
 }

 template<int T_D1D = 0, int T_Q1D = 0>
@@ -726,14 +741,21 @@ static void PAVectorDiffusionAssembleDiagonal(const int dim,

 void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   PAVectorDiffusionAssembleDiagonal(dim,
-                                     dofs1D,
-                                     quad1D,
-                                     ne,
-                                     maps->B,
-                                     maps->G,
-                                     pa_data,
-                                     diag);
+   if (DeviceCanUseCeed())
+   {
+      CeedAssembleDiagonal(ceedDataPtr, diag);
+   }
+   else
+   {
+      PAVectorDiffusionAssembleDiagonal(dim,
+                                        dofs1D,
+                                        quad1D,
+                                        ne,
+                                        maps->B,
+                                        maps->G,
+                                        pa_data,
+                                        diag);
+   }
 }

 } // namespace mfem
@@ -0,0 +1,69 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "libceed/diffusion.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+void VectorDiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
+{
+#ifdef MFEM_USE_CEED
+   // Assumes tensor-product elements
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement &el = *fes.GetFE(0);
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedMFDiffusionAssemble(fes, *ir, * ceedDataPtr);
+   }
+#endif
+   mfem_error("Error: VectorDiffusionIntegrator::AssembleMF only implemented with libCEED");
+}
+
+void VectorDiffusionIntegrator::AddMultMF(const Vector &x, Vector &y) const
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAddMult(ceedDataPtr, x, y);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: VectorDiffusionIntegrator::AssembleDiagonalMF only implemented with libCEED");
+   }
+}
+
+void VectorDiffusionIntegrator::AssembleDiagonalMF(Vector &diag)
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAssembleDiagonal(ceedDataPtr, diag);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: VectorDiffusionIntegrator::AddMultMF only implemented with libCEED");
+   }
+}
+
+} // namespace mfem
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "libceed/mass.hpp"

 using namespace std;

@@ -30,6 +31,13 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   ElementTransformation *T = mesh->GetElementTransformation(0);
   const IntegrationRule *ir
      = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedPAMassAssemble(fes, *ir, *ceedDataPtr);
+   }
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
@@ -361,7 +369,14 @@ static void PAVectorMassApply(const int dim,

 void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
+   if (DeviceCanUseCeed())
+   {
+      CeedAddMult(ceedDataPtr, x, y);
+   }
+   else
+   {
+      PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
+   }
 }

 template<const int T_D1D = 0, const int T_Q1D = 0>
@@ -514,14 +529,21 @@ static void PAVectorMassAssembleDiagonal(const int dim,

 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   PAVectorMassAssembleDiagonal(dim,
-                                dofs1D,
-                                quad1D,
-                                ne,
-                                maps->B,
-                                maps->Bt,
-                                pa_data,
-                                diag);
+   if (DeviceCanUseCeed())
+   {
+      CeedAssembleDiagonal(ceedDataPtr, diag);
+   }
+   else
+   {
+      PAVectorMassAssembleDiagonal(dim,
+                                   dofs1D,
+                                   quad1D,
+                                   ne,
+                                   maps->B,
+                                   maps->Bt,
+                                   pa_data,
+                                   diag);
+   }
 }

 } // namespace mfem
@@ -0,0 +1,74 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "libceed/mass.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// MF Mass Integrator
+
+// MF Mass Assemble kernel
+void VectorMassIntegrator::AssembleMF(const FiniteElementSpace &fes)
+{
+#ifdef MFEM_USE_CEED
+   // Assuming the same element type
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNE() == 0) { return; }
+   const FiniteElement &el = *fes.GetFE(0);
+   ElementTransformation *T = mesh->GetElementTransformation(0);
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedDataPtr;
+      ceedDataPtr = new CeedData;
+      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
+      return CeedMFMassAssemble(fes, *ir, *ceedDataPtr);
+   }
+#endif
+   mfem_error("Error: VectorMassIntegrator::AssembleMF only implemented with libCEED");
+}
+
+void VectorMassIntegrator::AddMultMF(const Vector &x, Vector &y) const
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAddMult(ceedDataPtr, x, y);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: VectorMassIntegrator::AssembleDiagonalMF only implemented with libCEED");
+   }
+}
+
+void VectorMassIntegrator::AssembleDiagonalMF(Vector &diag)
+{
+#ifdef MFEM_USE_CEED
+   if (DeviceCanUseCeed())
+   {
+      CeedAssembleDiagonal(ceedDataPtr, diag);
+   }
+   else
+#endif
+   {
+      mfem_error("Error: VectorMassIntegrator::AddMultMF only implemented with libCEED");
+   }
+}
+
+} // namespace mfem
@@ -15,21 +15,13 @@
 namespace mfem
 {

-void PAHcurlSetup2D(const int Q1D,
-                    const int coeffDim,
-                    const int NE,
-                    const Array<double> &w,
-                    const Vector &j,
-                    Vector &coeff,
-                    Vector &op);
-
-void PAHcurlSetup3D(const int Q1D,
-                    const int coeffDim,
-                    const int NE,
-                    const Array<double> &w,
-                    const Vector &j,
-                    Vector &coeff,
-                    Vector &op);
+void PADiffusionSetup3D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const Array<double> &w,
+                        const Vector &j,
+                        const Vector &_coeff,
+                        Vector &op);

 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
@@ -189,12 +181,11 @@ void PAHcurlHdivSetup3D(const int Q1D,
                        Vector &_coeff,
                        Vector &op)
 {
-   const int NQ = Q1D*Q1D*Q1D;
   const bool symmetric = (coeffDim != 9);
-   auto W = _w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), 9, NQ, NE);
+   auto W = Reshape(_w.Read(), Q1D, Q1D, Q1D);
+   auto J = Reshape(j.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, Q1D, Q1D, Q1D, NE);
+   auto y = Reshape(op.Write(), 9, Q1D, Q1D, Q1D, NE);

   const int i11 = 0;
   const int i12 = transpose ? 3 : 1;
@@ -206,83 +197,89 @@ void PAHcurlHdivSetup3D(const int Q1D,
   const int i32 = transpose ? 5 : 7;
   const int i33 = 8;

-   MFEM_FORALL(e, NE,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
-      for (int q = 0; q < NQ; ++q)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double detJ = J11 * (J22 * J33 - J32 * J23) -
-         /* */               J21 * (J12 * J33 - J32 * J13) +
-         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double w_detJ = W[q] / detJ;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-
-         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            // First compute entries of R = MJ
-            const double M11 = (!symmetric) ? coeff(i11, q, e) : coeff(0, q, e);
-            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
-            const double M13 = (!symmetric) ? coeff(i13, q, e) : coeff(2, q, e);
-            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
-            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(3, q, e);
-            const double M23 = (!symmetric) ? coeff(i23, q, e) : coeff(4, q, e);
-            const double M31 = (!symmetric) ? coeff(i31, q, e) : M13;
-            const double M32 = (!symmetric) ? coeff(i32, q, e) : M23;
-            const double M33 = (!symmetric) ? coeff(i33, q, e) : coeff(5, q, e);
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               const double J11 = J(qx,qy,qz,0,0,e);
+               const double J21 = J(qx,qy,qz,1,0,e);
+               const double J31 = J(qx,qy,qz,2,0,e);
+               const double J12 = J(qx,qy,qz,0,1,e);
+               const double J22 = J(qx,qy,qz,1,1,e);
+               const double J32 = J(qx,qy,qz,2,1,e);
+               const double J13 = J(qx,qy,qz,0,2,e);
+               const double J23 = J(qx,qy,qz,1,2,e);
+               const double J33 = J(qx,qy,qz,2,2,e);
+               const double detJ = J11 * (J22 * J33 - J32 * J23) -
+               /* */               J21 * (J12 * J33 - J32 * J13) +
+               /* */               J31 * (J12 * J23 - J22 * J13);
+               const double w_detJ = W(qx,qy,qz) / detJ;
+               // adj(J)
+               const double A11 = (J22 * J33) - (J23 * J32);
+               const double A12 = (J32 * J13) - (J12 * J33);
+               const double A13 = (J12 * J23) - (J22 * J13);
+               const double A21 = (J31 * J23) - (J21 * J33);
+               const double A22 = (J11 * J33) - (J13 * J31);
+               const double A23 = (J21 * J13) - (J11 * J23);
+               const double A31 = (J21 * J32) - (J31 * J22);
+               const double A32 = (J31 * J12) - (J11 * J32);
+               const double A33 = (J11 * J22) - (J12 * J21);

-            const double R11 = M11*J11 + M12*J12 + M13*J13;
-            const double R12 = M11*J21 + M12*J22 + M13*J23;
-            const double R13 = M11*J31 + M12*J32 + M13*J33;
-            const double R21 = M21*J11 + M22*J12 + M23*J13;
-            const double R22 = M21*J21 + M22*J22 + M23*J23;
-            const double R23 = M21*J31 + M22*J32 + M23*J33;
-            const double R31 = M31*J11 + M32*J12 + M33*J13;
-            const double R32 = M31*J21 + M32*J22 + M33*J23;
-            const double R33 = M31*J31 + M32*J32 + M33*J33;
+               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+               {
+                  // First compute entries of R = MJ
+                  const double M11 = (!symmetric) ? coeff(i11,qx,qy,qz,e) : coeff(0,qx,qy,qz,e);
+                  const double M12 = (!symmetric) ? coeff(i12,qx,qy,qz,e) : coeff(1,qx,qy,qz,e);
+                  const double M13 = (!symmetric) ? coeff(i13,qx,qy,qz,e) : coeff(2,qx,qy,qz,e);
+                  const double M21 = (!symmetric) ? coeff(i21,qx,qy,qz,e) : M12;
+                  const double M22 = (!symmetric) ? coeff(i22,qx,qy,qz,e) : coeff(3,qx,qy,qz,e);
+                  const double M23 = (!symmetric) ? coeff(i23,qx,qy,qz,e) : coeff(4,qx,qy,qz,e);
+                  const double M31 = (!symmetric) ? coeff(i31,qx,qy,qz,e) : M13;
+                  const double M32 = (!symmetric) ? coeff(i32,qx,qy,qz,e) : M23;
+                  const double M33 = (!symmetric) ? coeff(i33,qx,qy,qz,e) : coeff(5,qx,qy,qz,e);

-            // Now set y to detJ J^{-1} R = adj(J) R
-            y(i11,q,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-            y(i12,q,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
-            y(i13,q,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-            y(i21,q,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
-            y(i22,q,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
-            y(i23,q,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
-            y(i31,q,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-            y(i32,q,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-            y(i33,q,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
-         }
-         else if (coeffDim == 3)  // Vector coefficient version
-         {
-            const double D1 = coeff(0, q, e);
-            const double D2 = coeff(1, q, e);
-            const double D3 = coeff(2, q, e);
-            // detJ J^{-1} DJ = adj(J) DJ
-            y(i11,q,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
-            y(i12,q,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
-            y(i13,q,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
-            y(i21,q,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
-            y(i22,q,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
-            y(i23,q,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
-            y(i31,q,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
-            y(i32,q,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
-            y(i33,q,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
+                  const double R11 = M11*J11 + M12*J12 + M13*J13;
+                  const double R12 = M11*J21 + M12*J22 + M13*J23;
+                  const double R13 = M11*J31 + M12*J32 + M13*J33;
+                  const double R21 = M21*J11 + M22*J12 + M23*J13;
+                  const double R22 = M21*J21 + M22*J22 + M23*J23;
+                  const double R23 = M21*J31 + M22*J32 + M23*J33;
+                  const double R31 = M31*J11 + M32*J12 + M33*J13;
+                  const double R32 = M31*J21 + M32*J22 + M33*J23;
+                  const double R33 = M31*J31 + M32*J32 + M33*J33;
+
+                  // Now set y to detJ J^{-1} R = adj(J) R
+                  y(i11,qx,qy,qz,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+                  y(i12,qx,qy,qz,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
+                  y(i13,qx,qy,qz,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+                  y(i21,qx,qy,qz,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
+                  y(i22,qx,qy,qz,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
+                  y(i23,qx,qy,qz,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
+                  y(i31,qx,qy,qz,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+                  y(i32,qx,qy,qz,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+                  y(i33,qx,qy,qz,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
+               }
+               else if (coeffDim == 3)  // Vector coefficient version
+               {
+                  const double D1 = coeff(0,qx,qy,qz,e);
+                  const double D2 = coeff(1,qx,qy,qz,e);
+                  const double D3 = coeff(2,qx,qy,qz,e);
+                  // detJ J^{-1} DJ = adj(J) DJ
+                  y(i11,qx,qy,qz,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
+                  y(i12,qx,qy,qz,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
+                  y(i13,qx,qy,qz,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
+                  y(i21,qx,qy,qz,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
+                  y(i22,qx,qy,qz,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
+                  y(i23,qx,qy,qz,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
+                  y(i31,qx,qy,qz,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
+                  y(i32,qx,qy,qz,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
+                  y(i33,qx,qy,qz,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
+               }
+            }
         }
      }
   });
@@ -300,59 +297,61 @@ void PAHcurlHdivSetup2D(const int Q1D,
                        Vector &_coeff,
                        Vector &op)
 {
-   const int NQ = Q1D*Q1D;
   const bool symmetric = (coeffDim != 4);
-   auto W = _w.Read();
-   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), 4, NQ, NE);
+   auto W = Reshape(_w.Read(), Q1D, Q1D);
+   auto J = Reshape(j.Read(), Q1D, Q1D, 2, 2, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, Q1D, Q1D, NE);
+   auto y = Reshape(op.Write(), 4, Q1D, Q1D, NE);

   const int i11 = 0;
   const int i12 = transpose ? 2 : 1;
   const int i21 = transpose ? 1 : 2;
   const int i22 = 3;

-   MFEM_FORALL(e, NE,
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
   {
-      for (int q = 0; q < NQ; ++q)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double w_detJ = W[q] / (J11*J22) - (J21*J12);
-
-         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            // First compute entries of R = MJ
-            const double M11 = coeff(i11, q, e);
-            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
-            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
-            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(2, q, e);
+            const double J11 = J(qx,qy,0,0,e);
+            const double J21 = J(qx,qy,1,0,e);
+            const double J12 = J(qx,qy,0,1,e);
+            const double J22 = J(qx,qy,1,1,e);
+            const double w_detJ = W(qx,qy) / (J11*J22) - (J21*J12);

-            const double R11 = M11*J11 + M12*J21;
-            const double R12 = M11*J12 + M12*J22;
-            const double R21 = M21*J11 + M22*J21;
-            const double R22 = M21*J12 + M22*J22;
+            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
+            {
+               // First compute entries of R = MJ
+               const double M11 = coeff(i11,qx,qy,e);
+               const double M12 = (!symmetric) ? coeff(i12,qx,qy,e) : coeff(1,qx,qy,e);
+               const double M21 = (!symmetric) ? coeff(i21,qx,qy,e) : M12;
+               const double M22 = (!symmetric) ? coeff(i22,qx,qy,e) : coeff(2,qx,qy,e);

-            // Now set y to J^{-1} R
-            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-         }
-         else if (coeffDim == 2) // Vector coefficient version
-         {
-            const double D1 = coeff(0, q, e);
-            const double D2 = coeff(1, q, e);
-            const double R11 = D1*J11;
-            const double R12 = D1*J12;
-            const double R21 = D2*J21;
-            const double R22 = D2*J22;
-            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+               const double R11 = M11*J11 + M12*J21;
+               const double R12 = M11*J12 + M12*J22;
+               const double R21 = M21*J11 + M22*J21;
+               const double R22 = M21*J12 + M22*J22;
+
+               // Now set y to J^{-1} R
+               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+            }
+            else if (coeffDim == 2) // Vector coefficient version
+            {
+               const double D1 = coeff(0,qx,qy,e);
+               const double D2 = coeff(1,qx,qy,e);
+               const double R11 = D1*J11;
+               const double R12 = D1*J12;
+               const double R21 = D2*J21;
+               const double R22 = D2*J22;
+               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+            }
         }
      }
   });
@@ -856,13 +855,13 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   if (trial_curl && test_curl && dim == 3)
   {
-      PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
-                     coeff, pa_data);
+      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                         coeff, pa_data);
   }
   else if (trial_curl && test_curl && dim == 2)
   {
-      PAHcurlSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
-                     coeff, pa_data);
+      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                            coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 3)
   {
@@ -1109,13 +1108,13 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
   // Use the same setup functions as VectorFEMassIntegrator.
   if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
   {
-      PAHcurlSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
-                     coeff, pa_data);
+      PADiffusionSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                         coeff, pa_data);
   }
   else if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
   {
-      PAHcurlSetup2D(quad1D, 1, ne, ir->GetWeights(), geom->J,
-                     coeff, pa_data);
+      PADiffusionSetup2D<2>(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                            coeff, pa_data);
   }
   else
   {
@@ -38,11 +38,11 @@ double FunctionCoefficient::Eval(ElementTransformation & T,

   if (Function)
   {
-      return ((*Function)(transip));
+      return Function(transip);
   }
   else
   {
-      return (*TDFunction)(transip, GetTime());
+      return TDFunction(transip, GetTime());
   }
 }

@@ -112,11 +112,11 @@ void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
   V.SetSize(vdim);
   if (Function)
   {
-      (*Function)(transip, V);
+      Function(transip, V);
   }
   else
   {
-      (*TDFunction)(transip, GetTime(), V);
+      TDFunction(transip, GetTime(), V);
   }
   if (Q)
   {
@@ -301,18 +301,45 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,

   K.SetSize(height, width);

-   if (Function)
+   if (symmetric) // Use SymmFunction
   {
-      (*Function)(transip, K);
-   }
-   else if (TDFunction)
-   {
-      (*TDFunction)(transip, GetTime(), K);
+      MFEM_VERIFY(height == width && SymmFunction,
+                  "MatrixFunctionCoefficient is not symmetric");
+
+      Vector Ksym((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
+
+      SymmFunction(transip, Ksym);
+
+      // Copy upper triangular values from Ksym to the full matrix K
+      int os = 0;
+      for (int i=0; i<height; ++i)
+      {
+         for (int j=i; j<width; ++j)
+         {
+            const double Kij = Ksym[j - i + os];
+            K(i,j) = Kij;
+            if (j != i) { K(j,i) = Kij; }
+         }
+
+         os += width - i;
+      }
   }
   else
   {
-      K = mat;
+      if (Function)
+      {
+         Function(transip, K);
+      }
+      else if (TDFunction)
+      {
+         TDFunction(transip, GetTime(), K);
+      }
+      else
+      {
+         K = mat;
+      }
   }
+
   if (Q)
   {
      K *= Q->Eval(T, ip, GetTime());
@@ -323,7 +350,7 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
                                              ElementTransformation &T,
                                              const IntegrationPoint &ip)
 {
-   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
+   MFEM_VERIFY(symmetric && height == width && SymmFunction,
               "MatrixFunctionCoefficient is not symmetric");

   double x[3];
@@ -335,7 +362,7 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,

   if (SymmFunction)
   {
-      (*SymmFunction)(transip, K);
+      SymmFunction(transip, K);
   }

   if (Q)
@@ -12,6 +12,8 @@
 #ifndef MFEM_COEFFICIENT
 #define MFEM_COEFFICIENT

+#include <functional>
+
 #include "../config/config.hpp"
 #include "../linalg/linalg.hpp"
 #include "intrules.hpp"
@@ -123,28 +125,25 @@ public:
                       const IntegrationPoint &ip);
 };

-
-/// A general C-function coefficient
+/// A general function coefficient
 class FunctionCoefficient : public Coefficient
 {
 protected:
-   double (*Function)(const Vector &);
-   double (*TDFunction)(const Vector &, double);
+   std::function<double(const Vector &)> Function;
+   std::function<double(const Vector &, double)> TDFunction;

 public:
-   /// Define a time-independent coefficient from a pointer to a C-function
-   FunctionCoefficient(double (*f)(const Vector &))
-   {
-      Function = f;
-      TDFunction = NULL;
-   }
+   /// Define a time-independent coefficient from a std function
+   /** \param F time-independent std::function */
+   FunctionCoefficient(std::function<double(const Vector &)> F)
+      : Function(std::move(F))
+   { }

-   /// Define a time-dependent coefficient from a pointer to a C-function
-   FunctionCoefficient(double (*tdf)(const Vector &, double))
-   {
-      Function = NULL;
-      TDFunction = tdf;
-   }
+   /// Define a time-dependent coefficient from a std function
+   /** \param TDF time-dependent function */
+   FunctionCoefficient(std::function<double(const Vector &, double)> TDF)
+      : TDFunction(std::move(TDF))
+   { }

   /// (DEPRECATED) Define a time-independent coefficient from a C-function
   /** @deprecated Use the method where the C-function, @a f, uses a const
@@ -405,33 +404,34 @@ public:
   const Vector& GetVec() { return vec; }
 };

-/// A general C-function vector coefficient
+/// A general vector function coefficient
 class VectorFunctionCoefficient : public VectorCoefficient
 {
 private:
-   void (*Function)(const Vector &, Vector &);
-   void (*TDFunction)(const Vector &, double, Vector &);
+   std::function<void(const Vector &, Vector &)> Function;
+   std::function<void(const Vector &, double, Vector &)> TDFunction;
   Coefficient *Q;

 public:
-   /// Construct a time-independent vector coefficient from a C-function
-   VectorFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
-                             Coefficient *q = NULL)
-      : VectorCoefficient(dim), Q(q)
-   {
-      Function = F;
-      TDFunction = NULL;
-   }
-
-   /// Construct a time-dependent vector coefficient from a C-function
+   /// Define a time-independent vector coefficient from a std function
+   /** \param dim - the size of the vector
+       \param F - time-independent function
+       \param q - optional scalar Coefficient to scale the vector coefficient */
   VectorFunctionCoefficient(int dim,
-                             void (*TDF)(const Vector &, double, Vector &),
-                             Coefficient *q = NULL)
-      : VectorCoefficient(dim), Q(q)
-   {
-      Function = NULL;
-      TDFunction = TDF;
-   }
+                             std::function<void(const Vector &, Vector &)> F,
+                             Coefficient *q = nullptr)
+      : VectorCoefficient(dim), Function(std::move(F)), Q(q)
+   { }
+
+   /// Define a time-dependent vector coefficient from a std function
+   /** \param dim - the size of the vector
+       \param TDF - time-dependent function
+       \param q - optional scalar Coefficient to scale the vector coefficient */
+   VectorFunctionCoefficient(int dim,
+                             std::function<void(const Vector &, double, Vector &)> TDF,
+                             Coefficient *q = nullptr)
+      : VectorCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
+   { }

   using VectorCoefficient::Eval;
   /// Evaluate the vector coefficient at @a ip.
@@ -721,7 +721,6 @@ public:
   /// For backward compatibility get the width of the matrix.
   int GetVDim() const { return width; }

-   void SetSymmetric(bool s) { symmetric = s; }
   bool IsSymmetric() const { return symmetric; }

   /** @brief Evaluate the matrix coefficient in the element described by @a T
@@ -762,61 +761,56 @@ public:


 /** @brief A matrix coefficient with an optional scalar coefficient multiplier
-    \a q.  The matrix function can either be represented by a C-function or a
-    constant matrix provided when constructing this object.  */
+    \a q.  The matrix function can either be represented by a std function or
+    a constant matrix provided when constructing this object.  */
 class MatrixFunctionCoefficient : public MatrixCoefficient
 {
 private:
-   void (*Function)(const Vector &, DenseMatrix &);
-   void (*SymmFunction)(const Vector &, Vector &);
-   void (*TDFunction)(const Vector &, double, DenseMatrix &);
+   std::function<void(const Vector &, DenseMatrix &)> Function;
+   std::function<void(const Vector &, Vector &)> SymmFunction;
+   std::function<void(const Vector &, double, DenseMatrix &)> TDFunction;
+
   Coefficient *Q;
   DenseMatrix mat;

 public:
-   /// Construct a square matrix coefficient from a C-function without time
-   /// dependence.
-   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, DenseMatrix &),
-                             Coefficient *q = NULL)
-      : MatrixCoefficient(dim), Q(q)
-   {
-      Function = F;
-      TDFunction = NULL;
-      mat.SetSize(0);
-   }
-
-   /// Construct a constant matrix coefficient times a scalar Coefficient
-   MatrixFunctionCoefficient(const DenseMatrix &m, Coefficient &q)
-      : MatrixCoefficient(m.Height(), m.Width()), Q(&q)
-   {
-      Function = NULL;
-      TDFunction = NULL;
-      mat = m;
-   }
-
-   /// Construct a square matrix coefficient from a C-function with
-   /// time-dependence.
+   /// Define a time-independent square matrix coefficient from a std function
+   /** \param dim - the size of the matrix
+       \param F - time-independent function
+       \param q - optional scalar Coefficient to scale the matrix coefficient */
   MatrixFunctionCoefficient(int dim,
-                             void (*TDF)(const Vector &, double, DenseMatrix &),
-                             Coefficient *q = NULL)
-      : MatrixCoefficient(dim), Q(q)
-   {
-      Function = NULL;
-      TDFunction = TDF;
-      mat.SetSize(0);
-   }
+                             std::function<void(const Vector &, DenseMatrix &)> F,
+                             Coefficient *q = nullptr)
+      : MatrixCoefficient(dim), Function(std::move(F)), Q(q), mat(0)
+   { }

-   /// Construct a symmetric square matrix coefficient from a C-function
-   /// defining a vector function used by EvalSymmetric
-   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
+   /// Define a constant matrix coefficient times a scalar Coefficient
+   /** \param m - constant matrix
+       \param q - optional scalar Coefficient to scale the matrix coefficient */
+   MatrixFunctionCoefficient(const DenseMatrix &m, Coefficient &q)
+      : MatrixCoefficient(m.Height(), m.Width()), Q(&q), mat(m)
+   { }
+
+   /// Define a time-dependent square matrix coefficient from a std function
+   /** \param dim - the size of the matrix
+       \param TDF - time-dependent function
+       \param q - optional scalar Coefficient to scale the matrix coefficient */
+   MatrixFunctionCoefficient(int dim,
+                             std::function<void(const Vector &, double, DenseMatrix &)> TDF,
+                             Coefficient *q = nullptr)
+      : MatrixCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
+   { }
+
+   /** @brief Define a time-independent symmetric square matrix coefficient from
+       a std function */
+   /** \param dim - the size of the matrix
+       \param SymmF - function used in EvalSymmetric
+       \param q - optional scalar Coefficient to scale the matrix coefficient */
+   MatrixFunctionCoefficient(int dim,
+                             std::function<void(const Vector &, Vector &)> SymmF,
                             Coefficient *q = NULL)
-      : MatrixCoefficient(dim, true), Q(q)
-   {
-      SymmFunction = F;
-      Function = NULL;
-      TDFunction = NULL;
-      mat.SetSize(0);
-   }
+      : MatrixCoefficient(dim, true), SymmFunction(std::move(SymmF)), Q(q), mat(0)
+   { }

   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
@@ -889,7 +883,7 @@ public:

 /// Coefficients based on sums, products, or other functions of coefficients.
 ///@{
-/** Scalar coefficient defined as the linear combination of two scalar
+/** @brief Scalar coefficient defined as the linear combination of two scalar
    coefficients or a scalar and a scalar coefficient */
 class SumCoefficient : public Coefficient
 {
@@ -946,8 +940,8 @@ public:
   }
 };

-/** Scalar coefficient defined as the product of two scalar coefficients or
-    a scalar and a scalar coefficient. */
+/** @brief Scalar coefficient defined as the product of two scalar coefficients
+    or a scalar and a scalar coefficient. */
 class ProductCoefficient : public Coefficient
 {
 private:
@@ -985,8 +979,8 @@ public:
   { return ((a == NULL ) ? aConst : a->Eval(T, ip) ) * b->Eval(T, ip); }
 };

-/** Scalar coefficient defined as the ratio of two scalars where one or both
-    scalars are scalar coefficients. */
+/** @brief Scalar coefficient defined as the ratio of two scalars where one or
+    both scalars are scalar coefficients. */
 class RatioCoefficient : public Coefficient
 {
 private:
@@ -238,7 +238,8 @@ public:
   /// Set the desired assembly level.
   /** Valid choices are:

-       - AssemblyLevel::FULL  (default)
+       - AssemblyLevel::LEGACYFULL (default)
+       - AssemblyLevel::FULL
       - AssemblyLevel::PARTIAL
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE
@@ -575,7 +576,8 @@ public:
   /// Set the desired assembly level.
   /** Valid choices are:

-       - AssemblyLevel::FULL  (default)
+       - AssemblyLevel::LEGACYFULL (default)
+       - AssemblyLevel::FULL
       - AssemblyLevel::PARTIAL
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE
@@ -908,7 +908,7 @@ void ParaViewDataCollection::Save()

      // CELL DATA
      out << "<PCellData>\n";
-      out << "\t<PDataArray type=\"Int32\" Name=\"" << "material"
+      out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
          << "\" NumberOfComponents=\"1\""
          << " format=\"" << GetDataFormatString() << "\"/>\n";
      out << "</PCellData>\n";
@@ -382,6 +382,10 @@ public:
   int Error() const { return error; }
   /// Reset the error state
   void ResetError(int err = NO_ERROR) { error = err; }
+
+#ifdef MFEM_USE_MPI
+   friend class ParMesh;
+#endif
 };


@@ -14,6 +14,7 @@
 #include "../general/text.hpp"
 #include "../general/forall.hpp"
 #include "../mesh/mesh_headers.hpp"
+#include "../fem/libceed/ceed.hpp"
 #include "fem.hpp"

 #include <cmath>
@@ -698,7 +699,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
   for (int entity = 1; entity <= 2; entity++)
   {
      const NCMesh::NCList &list = mesh->ncmesh->GetNCList(entity);
-      if (!list.masters.size()) { continue; }
+      if (!list.masters.Size()) { continue; }

      Array<int> master_dofs, slave_dofs;

@@ -706,7 +707,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
      DenseMatrix I;

      // loop through all master edges/faces, constrain their slave edges/faces
-      for (unsigned mi = 0; mi < list.masters.size(); mi++)
+      for (int mi = 0; mi < list.masters.Size(); mi++)
      {
         const NCMesh::Master &master = list.masters[mi];

@@ -730,7 +731,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
            GetEntityDofs(entity, slave.index, slave_dofs, master.Geom());
            if (!slave_dofs.Size()) { continue; }

-            slave.OrientedPointMatrix(T.GetPointMat());
+            list.OrientedPointMatrix(slave, T.GetPointMat());
            fe->GetLocalInterpolation(T, I);

            // make each slave DOF dependent on all master DOFs
@@ -2160,6 +2161,7 @@ void FiniteElementSpace::Destroy()
      delete [] bdofs;
      delete [] fdofs;
   }
+   RemoveCeedBasisAndRestriction(this);
 }

 void FiniteElementSpace::GetTransferOperator(
@@ -68,7 +68,7 @@ void Hybridization::ConstructC()
      {
         const int dim = pmesh->Dimension();
         const NCMesh::NCList &shared = pmesh->pncmesh->GetSharedList(dim-1);
-         num_shared_slave_faces = (HYPRE_Int)shared.slaves.size();
+         num_shared_slave_faces = (HYPRE_Int) shared.slaves.Size();
         MPI_Allreduce(&num_shared_slave_faces, &glob_num_shared_slave_faces, 1,
                       HYPRE_MPI_INT, MPI_SUM, pmesh->GetComm());
         MFEM_ASSERT(glob_num_shared_slave_faces%2 == 0, "");
@@ -14,68 +14,29 @@

 #include "../../config/config.hpp"

-#ifdef MFEM_USE_CEED
 #include "../../general/device.hpp"
 #include "../../linalg/vector.hpp"
+#ifdef MFEM_USE_CEED
 #include <ceed.h>
 #include <ceed-hash.h>
+#endif
 #include <tuple>
 #include <unordered_map>

 namespace mfem
 {

+class Mesh;
 class FiniteElementSpace;
 class GridFunction;
 class IntegrationRule;
 class Coefficient;

-// Hash table for CeedBasis
-using CeedBasisKey =
-   std::tuple<const FiniteElementSpace*, const IntegrationRule*, int, int, int>;
-struct CeedBasisHash
-{
-   std::size_t operator()(const CeedBasisKey& k) const
-   {
-      return CeedHashCombine(CeedHashCombine(CeedHashInt(
-                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                                             CeedHashInt(
-                                                reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
-                             CeedHashCombine(CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                                             CeedHashInt(std::get<3>(k))),
-                                             CeedHashInt(std::get<4>(k))));
-   }
-};
-using CeedBasisMap =
-   std::unordered_map<const CeedBasisKey, CeedBasis, CeedBasisHash>;
-
-// Hash table for CeedElemRestriction
-using CeedRestrKey = std::tuple<const FiniteElementSpace*, int, int, int>;
-struct CeedRestrHash
-{
-   std::size_t operator()(const CeedRestrKey& k) const
-   {
-      return CeedHashCombine(CeedHashCombine(CeedHashInt(
-                                                reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
-                                             CeedHashInt(std::get<1>(k))),
-                             CeedHashCombine(CeedHashInt(std::get<2>(k)),
-                                             CeedHashInt(std::get<3>(k))));
-   }
-};
-using CeedRestrMap =
-   std::unordered_map<const CeedRestrKey, CeedElemRestriction, CeedRestrHash>;
-
-namespace internal
-{
-extern Ceed ceed; // defined in device.cpp
-extern CeedBasisMap basis_map;
-extern CeedRestrMap restr_map;
-}
-
+#ifdef MFEM_USE_CEED
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
-struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
+struct BuildContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };

-enum class CeedCoeff { Const, Grid };
+enum class CeedCoeff { Const, Grid, Quad };

 struct CeedConstCoeff
 {
@@ -90,8 +51,17 @@ struct CeedGridCoeff
   CeedVector coeffVector;
 };

+struct CeedQuadCoeff
+{
+   Vector coeff;
+   CeedElemRestriction restr;
+   CeedVector coeffVector;
+};
+#endif
+
 struct CeedData
 {
+#ifdef MFEM_USE_CEED
   CeedOperator build_oper, oper;
   CeedBasis basis, mesh_basis;
   CeedElemRestriction restr, mesh_restr, restr_i, mesh_restr_i;
@@ -104,39 +74,51 @@ struct CeedData

   CeedVector u, v;

+   CeedData()
+      : build_oper(nullptr), oper(nullptr), basis(nullptr), mesh_basis(nullptr),
+        restr(nullptr), mesh_restr(nullptr), restr_i(nullptr), mesh_restr_i(nullptr),
+        apply_qfunc(nullptr), build_qfunc(nullptr), node_coords(nullptr),
+        rho(nullptr), coeff(nullptr), build_ctx(nullptr) { }
+
   ~CeedData()
   {
      CeedOperatorDestroy(&build_oper);
      CeedOperatorDestroy(&oper);
-      CeedElemRestrictionDestroy(&restr_i);
-      CeedElemRestrictionDestroy(&mesh_restr_i);
      CeedQFunctionDestroy(&apply_qfunc);
      CeedQFunctionDestroy(&build_qfunc);
+      CeedQFunctionContextDestroy(&build_ctx);
      CeedVectorDestroy(&node_coords);
      CeedVectorDestroy(&rho);
-      if (coeff_type==CeedCoeff::Grid)
+      if (coeff_type==CeedCoeff::Const)
      {
-         CeedGridCoeff* c = (CeedGridCoeff*)coeff;
+         delete static_cast<CeedConstCoeff*>(coeff);
+      }
+      else if (coeff_type==CeedCoeff::Grid)
+      {
+         CeedGridCoeff* c = static_cast<CeedGridCoeff*>(coeff);
         CeedVectorDestroy(&c->coeffVector);
         delete c;
      }
-      else
+      else if (coeff_type==CeedCoeff::Quad)
      {
-         delete (CeedConstCoeff*)coeff;
+         CeedQuadCoeff* c = static_cast<CeedQuadCoeff*>(coeff);
+         CeedVectorDestroy(&c->coeffVector);
+         delete c;
      }
      CeedVectorDestroy(&u);
      CeedVectorDestroy(&v);
   }
-
+#endif
 };

 /** This structure contains the data to assemble a PA operator with libCEED.
    See libceed/mass.cpp or libceed/diffusion.cpp for examples. */
 struct CeedPAOperator
 {
+#ifdef MFEM_USE_CEED
   /** The finite element space for the trial and test functions. */
   const FiniteElementSpace &fes;
-   /** The Integration Rule to use to compote the operator. */
+   /** The Integration Rule to use to compute the operator. */
   const IntegrationRule &ir;
   /** The number of quadrature data at each quadrature point. */
   int qdatasize;
@@ -147,11 +129,11 @@ struct CeedPAOperator
   std::string const_func;
   /** The Qfunction to build the quadrature data with constant coefficient. */
   CeedQFunctionUser const_qf;
-   /** The name of the Qfunction to build the quadrature data with grid function
-       coefficient. */
-   std::string grid_func;
-   /** The Qfunction to build the quad. data with grid function coefficient. */
-   CeedQFunctionUser grid_qf;
+   /** The name of the Qfunction to build the quadrature data with a coefficient
+       evaluated at quadrature points. */
+   std::string quad_func;
+   /** The Qfunction to build the quad. data with a coefficient. */
+   CeedQFunctionUser quad_qf;
   /** The name of the Qfunction to apply the operator. */
   std::string apply_func;
   /** The Qfunction to apply the operator. */
@@ -162,11 +144,84 @@ struct CeedPAOperator
   /** The evaluation mode to apply to the test function ( CEED_EVAL_INTERP,
       CEED_EVAL_GRAD, etc.)*/
   CeedEvalMode test_op;
+#endif
+};
+
+struct CeedMFOperator
+{
+#ifdef MFEM_USE_CEED
+   /** The finite element space for the trial and test functions. */
+   const FiniteElementSpace &fes;
+   /** The Integration Rule to use to compote the operator. */
+   const IntegrationRule &ir;
+   /** The path to the header containing the functions for libCEED. */
+   std::string header;
+   /** The name of the Qfunction to apply the quadrature data with a constant
+       coefficient.*/
+   std::string const_func;
+   /** The Qfunction to apply the quadrature data with constant coefficient. */
+   CeedQFunctionUser const_qf;
+   /** The name of the Qfunction to apply the quadrature data with a coefficient
+       evaluated at quadrature point. */
+   std::string quad_func;
+   /** The Qfunction to apply the quad. data with a coefficient evaluated at
+       quadrature point. */
+   CeedQFunctionUser quad_qf;
+   /** The evaluation mode to apply to the trial function (CEED_EVAL_INTERP,
+       CEED_EVAL_GRAD, etc.) */
+   CeedEvalMode trial_op;
+   /** The evaluation mode to apply to the test function ( CEED_EVAL_INTERP,
+       CEED_EVAL_GRAD, etc.)*/
+   CeedEvalMode test_op;
+#endif
 };

 /** @brief Identifies the type of coefficient of the Integrator to initialize
    accordingly the CeedData. */
-void InitCeedCoeff(Coefficient* Q, CeedData* ptr);
+void InitCeedCoeff(Coefficient* Q, Mesh &mesh, const IntegrationRule &ir,
+                   CeedData* ptr);
+
+/** This function initializes an arbitrary linear operator using the partial
+    assembly decomposition in libCEED. The operator details are described by the
+    struct CEEDPAOperator input. */
+void CeedPAAssemble(const CeedPAOperator& op,
+                    CeedData& ceedData);
+
+/** This function initializes an arbitrary linear operator using a fully matrix
+    free decomposition. The operator details are described by the
+    struct CEEDPAOperator input. */
+void CeedMFAssemble(const CeedMFOperator& op,
+                    CeedData& ceedData);
+
+/** @brief Function that applies a libCEED operator. */
+void CeedAddMult(const CeedData *ceedDataPtr,
+                 const Vector &x,
+                 Vector &y);
+
+/** @brief Function that assembles a libCEED operator diagonal. */
+void CeedAssembleDiagonal(const CeedData *ceedDataPtr,
+                          Vector &diag);
+
+/** @brief Function that determines if a CEED kernel should be used, based on
+    the current mfem::Device configuration. */
+inline bool DeviceCanUseCeed()
+{
+   return Device::Allows(Backend::CEED_MASK);
+}
+
+/** @brief Remove from ceed_basis_map and ceed_restr_map the entries associated
+    with the given @a fes. */
+void RemoveCeedBasisAndRestriction(const FiniteElementSpace *fes);
+
+#ifdef MFEM_USE_CEED
+/// Initialize a CeedVector from a Vector
+void InitCeedVector(const Vector &v, CeedVector &cv);
+
+/// Initialize a strided CeedElemRestriction
+void InitCeedStridedRestriction(const FiniteElementSpace &fes,
+                                CeedInt nelem, CeedInt nqpts, CeedInt qdatasize,
+                                const CeedInt *strides,
+                                CeedElemRestriction *restr);

 /// Initialize a CeedBasis and a CeedElemRestriction
 void InitCeedBasisAndRestriction(const FiniteElementSpace &fes,
@@ -177,43 +232,52 @@ void InitCeedBasisAndRestriction(const FiniteElementSpace &fes,
 /// Return the path to the libCEED q-function headers.
 const std::string &GetCeedPath();

-/** This function initializes an arbitrary linear operator using the partial
-    assembly decomposition in libCEED. The operator details are described by the
-    struct CEEDPAOperator input. */
-void CeedPAAssemble(const CeedPAOperator& op,
-                    CeedData& ceedData);
-
-/** @brief Function that applies a libCEED PA operator. */
-void CeedAddMultPA(const CeedData *ceedDataPtr,
-                   const Vector &x,
-                   Vector &y);
-
-/** @brief Function that assembles a libCEED PA operator diagonal. */
-void CeedAssembleDiagonalPA(const CeedData *ceedDataPtr,
-                            Vector &diag);
-
-/** @brief Function that determines if a CEED kernel should be used, based on
-    the current mfem::Device configuration. */
-inline bool DeviceCanUseCeed()
+// Hash table for CeedBasis
+using CeedBasisKey =
+   std::tuple<const FiniteElementSpace*, const IntegrationRule*, int, int, int>;
+struct CeedBasisHash
 {
-   return Device::Allows(Backend::CEED_CUDA) ||
-          (Device::Allows(Backend::CEED_CPU) &&
-           !Device::Allows(Backend::DEVICE_MASK|Backend::OMP_MASK));
+   std::size_t operator()(const CeedBasisKey& k) const
+   {
+      return CeedHashCombine(
+                CeedHashCombine(
+                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
+                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<1>(k)))),
+                CeedHashCombine(
+                   CeedHashCombine(CeedHashInt(std::get<2>(k)),
+                                   CeedHashInt(std::get<3>(k))),
+                   CeedHashInt(std::get<4>(k))));
+   }
+};
+using CeedBasisMap =
+   std::unordered_map<const CeedBasisKey, CeedBasis, CeedBasisHash>;
+
+// Hash table for CeedElemRestriction
+using CeedRestrKey = std::tuple<const FiniteElementSpace*, int, int, int>;
+struct CeedRestrHash
+{
+   std::size_t operator()(const CeedRestrKey& k) const
+   {
+      return CeedHashCombine(
+                CeedHashCombine(
+                   CeedHashInt(reinterpret_cast<CeedHash64_t>(std::get<0>(k))),
+                   CeedHashInt(std::get<1>(k))),
+                CeedHashCombine(CeedHashInt(std::get<2>(k)),
+                                CeedHashInt(std::get<3>(k))));
+   }
+};
+using CeedRestrMap =
+   std::unordered_map<const CeedRestrKey, CeedElemRestriction, CeedRestrHash>;
+
+namespace internal
+{
+extern Ceed ceed; // defined in device.cpp
+extern CeedBasisMap basis_map;
+extern CeedRestrMap restr_map;
 }

-} // namespace mfem
-
-#else // MFEM_USE_CEED
-
-namespace mfem
-{
-inline bool DeviceCanUseCeed()
-{
-   return false;
-}
-
-} // namespace mfem
-
 #endif // MFEM_USE_CEED

+} // namespace mfem
+
 #endif // MFEM_LIBCEED_HPP
@@ -11,9 +11,10 @@

 #include "diffusion.hpp"

-#ifdef MFEM_USE_CEED
 #include "ceed.hpp"
+#ifdef MFEM_USE_CEED
 #include "diffusion.h"
+#endif

 namespace mfem
 {
@@ -21,18 +22,42 @@ namespace mfem
 void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
                             const mfem::IntegrationRule &irm, CeedData& ceedData)
 {
-   CeedInt dim = fes.GetMesh()->SpaceDimension();
+#ifdef MFEM_USE_CEED
+   Mesh &mesh = *fes.GetMesh();
+   // Perform checks for some assumptions made in the Q-functions.
+   MFEM_VERIFY(mesh.Dimension() == mesh.SpaceDimension(), "case not supported");
+   MFEM_VERIFY(fes.GetVDim() == 1 || fes.GetVDim() == mesh.Dimension(),
+               "case not supported");
+   int dim = mesh.Dimension();
   CeedPAOperator diffOp = {fes, irm,
                            dim * (dim + 1) / 2, "/diffusion.h",
                            ":f_build_diff_const", f_build_diff_const,
-                            ":f_build_diff_grid", f_build_diff_grid,
+                            ":f_build_diff_quad", f_build_diff_quad,
                            ":f_apply_diff", f_apply_diff,
                            CEED_EVAL_GRAD,
                            CEED_EVAL_GRAD
                           };
   CeedPAAssemble(diffOp, ceedData);
+#else
+   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+void CeedMFDiffusionAssemble(const FiniteElementSpace &fes,
+                             const mfem::IntegrationRule &irm, CeedData& ceedData)
+{
+#ifdef MFEM_USE_CEED
+   CeedMFOperator diffOp = {fes, irm,
+                            "/diffusion.h",
+                            ":f_apply_diff_mf_const", f_apply_diff_mf_const,
+                            ":f_apply_diff_mf_quad", f_apply_diff_mf_quad,
+                            CEED_EVAL_GRAD,
+                            CEED_EVAL_GRAD
+                           };
+   CeedMFAssemble(diffOp, ceedData);
+#else
+   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
 }

 } // namespace mfem
-
-#endif // MFEM_USE_CEED
@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.

 /// A structure used to pass additional data to f_build_diff and f_apply_diff
-struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
+struct BuildContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };

 /// libCEED Q-function for building quadrature data for a diffusion operator
 /// with a constant coefficient
@@ -88,8 +88,8 @@ CEED_QFUNCTION(f_build_diff_const)(void *ctx, CeedInt Q,
 }

 /// libCEED Q-function for building quadrature data for a diffusion operator
-/// with a grid function coefficient
-CEED_QFUNCTION(f_build_diff_grid)(void *ctx, CeedInt Q,
+/// coefficient evaluated at quadrature points.
+CEED_QFUNCTION(f_build_diff_quad)(void *ctx, CeedInt Q,
                                  const CeedScalar *const *in,
                                  CeedScalar *const *out)
 {
@@ -172,15 +172,15 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
   // in[0], out[0] have shape [dim, nc=1, Q]
   const CeedScalar *ug = in[0], *qd = in[1];
   CeedScalar *vg = out[0];
-   switch (bc->dim)
+   switch (10*bc->dim + bc->vdim)
   {
-      case 1:
+      case 11:
         for (CeedInt i = 0; i < Q; i++)
         {
            vg[i] = ug[i] * qd[i];
         }
         break;
-      case 2:
+      case 21:
         for (CeedInt i = 0; i < Q; i++)
         {
            const CeedScalar ug0 = ug[i + Q * 0];
@@ -189,7 +189,23 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
            vg[i + Q * 1] = qd[i + Q * 1] * ug0 + qd[i + Q * 2] * ug1;
         }
         break;
-      case 3:
+      case 22:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            const CeedScalar qd00 = qd[i + Q * 0];
+            const CeedScalar qd01 = qd[i + Q * 1];
+            const CeedScalar qd10 = qd01;
+            const CeedScalar qd11 = qd[i + Q * 2];
+            for (CeedInt c = 0; c < 2; c++)
+            {
+               const CeedScalar ug0 = ug[i + Q * (c+2*0)];
+               const CeedScalar ug1 = ug[i + Q * (c+2*1)];
+               vg[i + Q * (c+2*0)] = qd00 * ug0 + qd01 * ug1;
+               vg[i + Q * (c+2*1)] = qd10 * ug0 + qd11 * ug1;
+            }
+         }
+         break;
+      case 31:
         for (CeedInt i = 0; i < Q; i++)
         {
            const CeedScalar ug0 = ug[i + Q * 0];
@@ -200,6 +216,337 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
            vg[i + Q * 2] = qd[i + Q * 2] * ug0 + qd[i + Q * 4] * ug1 + qd[i + Q * 5] * ug2;
         }
         break;
+      case 33:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            const CeedScalar qd00 = qd[i + Q * 0];
+            const CeedScalar qd01 = qd[i + Q * 1];
+            const CeedScalar qd02 = qd[i + Q * 2];
+            const CeedScalar qd10 = qd01;
+            const CeedScalar qd11 = qd[i + Q * 3];
+            const CeedScalar qd12 = qd[i + Q * 4];
+            const CeedScalar qd20 = qd02;
+            const CeedScalar qd21 = qd12;
+            const CeedScalar qd22 = qd[i + Q * 5];
+            for (CeedInt c = 0; c < 3; c++)
+            {
+               const CeedScalar ug0 = ug[i + Q * (c+3*0)];
+               const CeedScalar ug1 = ug[i + Q * (c+3*1)];
+               const CeedScalar ug2 = ug[i + Q * (c+3*2)];
+               vg[i + Q * (c+3*0)] = qd00 * ug0 + qd01 * ug1 + qd02 * ug2;
+               vg[i + Q * (c+3*1)] = qd10 * ug0 + qd11 * ug1 + qd12 * ug2;
+               vg[i + Q * (c+3*2)] = qd20 * ug0 + qd21 * ug1 + qd22 * ug2;
+            }
+         }
+         break;
+   }
+   return 0;
+}
+
+/// libCEED Q-function for applying a diff operator
+CEED_QFUNCTION(f_apply_diff_mf_const)(void *ctx, CeedInt Q,
+                                      const CeedScalar *const *in,
+                                      CeedScalar *const *out)
+{
+   BuildContext *bc = (BuildContext*)ctx;
+   // in[0], out[0] have shape [dim, nc=1, Q]
+   // in[1] is Jacobians with shape [dim, nc=dim, Q]
+   // in[2] is quadrature weights, size (Q)
+   //
+   // At every quadrature point, compute qw/det(J).adj(J).adj(J)^T
+   const CeedScalar coeff = bc->coeff;
+   const CeedScalar *ug = in[0], *J = in[1], *qw = in[2];
+   CeedScalar *vg = out[0];
+   switch (10 * bc->dim + bc->vdim)
+   {
+      case 11:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            const CeedScalar qd = coeff * qw[i] / J[i];
+            vg[i] = ug[i] * qd;
+         }
+         break;
+      case 21:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
+            //    1 3       1 2           -J21  J11
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J12 = J[i + Q * 2];
+            const CeedScalar J22 = J[i + Q * 3];
+            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
+            CeedScalar qd[3];
+            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
+            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
+            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
+            const CeedScalar ug0 = ug[i + Q * 0];
+            const CeedScalar ug1 = ug[i + Q * 1];
+            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1;
+            vg[i + Q * 1] = qd[1] * ug0 + qd[2] * ug1;
+         }
+         break;
+      case 22:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
+            //    1 3       1 2           -J21  J11
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J12 = J[i + Q * 2];
+            const CeedScalar J22 = J[i + Q * 3];
+            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
+            CeedScalar qd[3];
+            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
+            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
+            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
+            for (CeedInt c = 0; c < 2; c++)
+            {
+               const CeedScalar ug0 = ug[i + Q * (c+2*0)];
+               const CeedScalar ug1 = ug[i + Q * (c+2*1)];
+               vg[i + Q * (c+2*0)] = qd[0] * ug0 + qd[1] * ug1;
+               vg[i + Q * (c+2*1)] = qd[1] * ug0 + qd[2] * ug1;
+            }
+         }
+         break;
+      case 31:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 3 6   qd: 0 1 2
+            //    1 4 7       1 3 4
+            //    2 5 8       2 4 5
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J31 = J[i + Q * 2];
+            const CeedScalar J12 = J[i + Q * 3];
+            const CeedScalar J22 = J[i + Q * 4];
+            const CeedScalar J32 = J[i + Q * 5];
+            const CeedScalar J13 = J[i + Q * 6];
+            const CeedScalar J23 = J[i + Q * 7];
+            const CeedScalar J33 = J[i + Q * 8];
+            const CeedScalar A11 = J22 * J33 - J23 * J32;
+            const CeedScalar A12 = J13 * J32 - J12 * J33;
+            const CeedScalar A13 = J12 * J23 - J13 * J22;
+            const CeedScalar A21 = J23 * J31 - J21 * J33;
+            const CeedScalar A22 = J11 * J33 - J13 * J31;
+            const CeedScalar A23 = J13 * J21 - J11 * J23;
+            const CeedScalar A31 = J21 * J32 - J22 * J31;
+            const CeedScalar A32 = J12 * J31 - J11 * J32;
+            const CeedScalar A33 = J11 * J22 - J12 * J21;
+            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
+            CeedScalar qd[6];
+            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
+            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
+            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
+            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
+            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
+            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
+            const CeedScalar ug0 = ug[i + Q * 0];
+            const CeedScalar ug1 = ug[i + Q * 1];
+            const CeedScalar ug2 = ug[i + Q * 2];
+            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
+            vg[i + Q * 1] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
+            vg[i + Q * 2] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
+         }
+         break;
+      case 33:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 3 6   qd: 0 1 2
+            //    1 4 7       1 3 4
+            //    2 5 8       2 4 5
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J31 = J[i + Q * 2];
+            const CeedScalar J12 = J[i + Q * 3];
+            const CeedScalar J22 = J[i + Q * 4];
+            const CeedScalar J32 = J[i + Q * 5];
+            const CeedScalar J13 = J[i + Q * 6];
+            const CeedScalar J23 = J[i + Q * 7];
+            const CeedScalar J33 = J[i + Q * 8];
+            const CeedScalar A11 = J22 * J33 - J23 * J32;
+            const CeedScalar A12 = J13 * J32 - J12 * J33;
+            const CeedScalar A13 = J12 * J23 - J13 * J22;
+            const CeedScalar A21 = J23 * J31 - J21 * J33;
+            const CeedScalar A22 = J11 * J33 - J13 * J31;
+            const CeedScalar A23 = J13 * J21 - J11 * J23;
+            const CeedScalar A31 = J21 * J32 - J22 * J31;
+            const CeedScalar A32 = J12 * J31 - J11 * J32;
+            const CeedScalar A33 = J11 * J22 - J12 * J21;
+            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
+            CeedScalar qd[6];
+            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
+            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
+            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
+            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
+            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
+            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
+            for (CeedInt c = 0; c < 3; c++)
+            {
+               const CeedScalar ug0 = ug[i + Q * (c+3*0)];
+               const CeedScalar ug1 = ug[i + Q * (c+3*1)];
+               const CeedScalar ug2 = ug[i + Q * (c+3*2)];
+               vg[i + Q * (c+3*0)] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
+               vg[i + Q * (c+3*1)] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
+               vg[i + Q * (c+3*2)] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
+            }
+         }
+         break;
+   }
+   return 0;
+}
+
+CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
+                                     const CeedScalar *const *in,
+                                     CeedScalar *const *out)
+{
+   BuildContext *bc = (BuildContext*)ctx;
+   // in[0], out[0] have shape [dim, nc=1, Q]
+   // in[1] is Jacobians with shape [dim, nc=dim, Q]
+   // in[2] is quadrature weights, size (Q)
+   //
+   // At every quadrature point, compute qw/det(J).adj(J).adj(J)^T
+   const CeedScalar *c = in[0], *ug = in[1], *J = in[2], *qw = in[3];
+   CeedScalar *vg = out[0];
+   switch (10 * bc->dim + bc->vdim)
+   {
+      case 11:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            const CeedScalar qd = c[i] * qw[i] / J[i];
+            vg[i] = ug[i] * qd;
+         }
+         break;
+      case 21:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
+            //    1 3       1 2           -J21  J11
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J12 = J[i + Q * 2];
+            const CeedScalar J22 = J[i + Q * 3];
+            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
+            CeedScalar qd[3];
+            const CeedScalar coeff = c[i];
+            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
+            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
+            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
+            const CeedScalar ug0 = ug[i + Q * 0];
+            const CeedScalar ug1 = ug[i + Q * 1];
+            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1;
+            vg[i + Q * 1] = qd[1] * ug0 + qd[2] * ug1;
+         }
+         break;
+      case 22:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 2   qd: 0 1   adj(J):  J22 -J12
+            //    1 3       1 2           -J21  J11
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J12 = J[i + Q * 2];
+            const CeedScalar J22 = J[i + Q * 3];
+            const CeedScalar w = qw[i] / (J11 * J22 - J21 * J12);
+            CeedScalar qd[3];
+            const CeedScalar coeff = c[i];
+            qd[0] =   coeff * w * (J12 * J12 + J22 * J22);
+            qd[1] = - coeff * w * (J11 * J12 + J21 * J22);
+            qd[2] =   coeff * w * (J11 * J11 + J21 * J21);
+            for (CeedInt c = 0; c < 2; c++)
+            {
+               const CeedScalar ug0 = ug[i + Q * (c+2*0)];
+               const CeedScalar ug1 = ug[i + Q * (c+2*1)];
+               vg[i + Q * (c+2*0)] = qd[0] * ug0 + qd[1] * ug1;
+               vg[i + Q * (c+2*1)] = qd[1] * ug0 + qd[2] * ug1;
+            }
+         }
+         break;
+      case 31:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 3 6   qd: 0 1 2
+            //    1 4 7       1 3 4
+            //    2 5 8       2 4 5
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J31 = J[i + Q * 2];
+            const CeedScalar J12 = J[i + Q * 3];
+            const CeedScalar J22 = J[i + Q * 4];
+            const CeedScalar J32 = J[i + Q * 5];
+            const CeedScalar J13 = J[i + Q * 6];
+            const CeedScalar J23 = J[i + Q * 7];
+            const CeedScalar J33 = J[i + Q * 8];
+            const CeedScalar A11 = J22 * J33 - J23 * J32;
+            const CeedScalar A12 = J13 * J32 - J12 * J33;
+            const CeedScalar A13 = J12 * J23 - J13 * J22;
+            const CeedScalar A21 = J23 * J31 - J21 * J33;
+            const CeedScalar A22 = J11 * J33 - J13 * J31;
+            const CeedScalar A23 = J13 * J21 - J11 * J23;
+            const CeedScalar A31 = J21 * J32 - J22 * J31;
+            const CeedScalar A32 = J12 * J31 - J11 * J32;
+            const CeedScalar A33 = J11 * J22 - J12 * J21;
+            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
+            CeedScalar qd[6];
+            const CeedScalar coeff = c[i];
+            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
+            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
+            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
+            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
+            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
+            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
+            const CeedScalar ug0 = ug[i + Q * 0];
+            const CeedScalar ug1 = ug[i + Q * 1];
+            const CeedScalar ug2 = ug[i + Q * 2];
+            vg[i + Q * 0] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
+            vg[i + Q * 1] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
+            vg[i + Q * 2] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
+         }
+         break;
+      case 33:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            // J: 0 3 6   qd: 0 1 2
+            //    1 4 7       1 3 4
+            //    2 5 8       2 4 5
+            const CeedScalar J11 = J[i + Q * 0];
+            const CeedScalar J21 = J[i + Q * 1];
+            const CeedScalar J31 = J[i + Q * 2];
+            const CeedScalar J12 = J[i + Q * 3];
+            const CeedScalar J22 = J[i + Q * 4];
+            const CeedScalar J32 = J[i + Q * 5];
+            const CeedScalar J13 = J[i + Q * 6];
+            const CeedScalar J23 = J[i + Q * 7];
+            const CeedScalar J33 = J[i + Q * 8];
+            const CeedScalar A11 = J22 * J33 - J23 * J32;
+            const CeedScalar A12 = J13 * J32 - J12 * J33;
+            const CeedScalar A13 = J12 * J23 - J13 * J22;
+            const CeedScalar A21 = J23 * J31 - J21 * J33;
+            const CeedScalar A22 = J11 * J33 - J13 * J31;
+            const CeedScalar A23 = J13 * J21 - J11 * J23;
+            const CeedScalar A31 = J21 * J32 - J22 * J31;
+            const CeedScalar A32 = J12 * J31 - J11 * J32;
+            const CeedScalar A33 = J11 * J22 - J12 * J21;
+            const CeedScalar w = qw[i] / (J11 * A11 + J21 * A12 + J31 * A13);
+            CeedScalar qd[6];
+            const CeedScalar coeff = c[i];
+            qd[0] = coeff * w * (A11 * A11 + A12 * A12 + A13 * A13);
+            qd[1] = coeff * w * (A11 * A21 + A12 * A22 + A13 * A23);
+            qd[2] = coeff * w * (A11 * A31 + A12 * A32 + A13 * A33);
+            qd[3] = coeff * w * (A21 * A21 + A22 * A22 + A23 * A23);
+            qd[4] = coeff * w * (A21 * A31 + A22 * A32 + A23 * A33);
+            qd[5] = coeff * w * (A31 * A31 + A32 * A32 + A33 * A33);
+            for (CeedInt c = 0; c < 3; c++)
+            {
+               const CeedScalar ug0 = ug[i + Q * (c+3*0)];
+               const CeedScalar ug1 = ug[i + Q * (c+3*1)];
+               const CeedScalar ug2 = ug[i + Q * (c+3*2)];
+               vg[i + Q * (c+3*0)] = qd[0] * ug0 + qd[1] * ug1 + qd[2] * ug2;
+               vg[i + Q * (c+3*1)] = qd[1] * ug0 + qd[3] * ug1 + qd[4] * ug2;
+               vg[i + Q * (c+3*2)] = qd[2] * ug0 + qd[4] * ug1 + qd[5] * ug2;
+            }
+         }
+         break;
   }
   return 0;
 }
@@ -14,7 +14,6 @@

 #include "ceed.hpp"

-#ifdef MFEM_USE_CEED
 #include "../fespace.hpp"

 namespace mfem
@@ -25,8 +24,10 @@ void CeedPADiffusionAssemble(const FiniteElementSpace &fes,
                             const mfem::IntegrationRule &ir,
                             CeedData& ceedData);

+void CeedMFDiffusionAssemble(const FiniteElementSpace &fes,
+                             const mfem::IntegrationRule &ir,
+                             CeedData& ceedData);
+
 }

-#endif // MFEM_USE_CEED
-
 #endif // MFEM_LIBCEED_DIFF_HPP
@@ -11,9 +11,10 @@

 #include "mass.hpp"

-#ifdef MFEM_USE_CEED
 #include "ceed.hpp"
+#ifdef MFEM_USE_CEED
 #include "mass.h"
+#endif

 namespace mfem
 {
@@ -21,17 +22,41 @@ namespace mfem
 void CeedPAMassAssemble(const FiniteElementSpace &fes,
                        const mfem::IntegrationRule &irm, CeedData& ceedData)
 {
+#ifdef MFEM_USE_CEED
+   Mesh &mesh = *fes.GetMesh();
+   // Perform checks for some assumptions made in the Q-functions.
+   MFEM_VERIFY(mesh.Dimension() == mesh.SpaceDimension(), "case not supported");
+   MFEM_VERIFY(1 <= fes.GetVDim() && fes.GetVDim() <= 3, "case not supported");
   CeedPAOperator massOp = {fes, irm,
                            1, "/mass.h",
                            ":f_build_mass_const", f_build_mass_const,
-                            ":f_build_mass_grid", f_build_mass_grid,
+                            ":f_build_mass_quad", f_build_mass_quad,
                            ":f_apply_mass", f_apply_mass,
                            CEED_EVAL_INTERP,
                            CEED_EVAL_INTERP
                           };
   CeedPAAssemble(massOp, ceedData);
+#else
+   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+void CeedMFMassAssemble(const FiniteElementSpace &fes,
+                        const mfem::IntegrationRule &irm, CeedData& ceedData)
+{
+#ifdef MFEM_USE_CEED
+   CeedMFOperator massOp = {fes, irm,
+                            "/mass.h",
+                            ":f_apply_mass_mf_const", f_apply_mass_mf_const,
+                            ":f_apply_mass_mf_quad", f_apply_mass_mf_quad,
+                            CEED_EVAL_INTERP,
+                            CEED_EVAL_INTERP
+                           };
+   CeedMFAssemble(massOp, ceedData);
+#else
+   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
 }

 } // namespace mfem

-#endif // MFEM_USE_CEED
@@ -10,7 +10,7 @@
 // CONTRIBUTING.md for details.

 /// A structure used to pass additional data to f_build_diff and f_apply_diff
-struct BuildContext { CeedInt dim, space_dim; CeedScalar coeff; };
+struct BuildContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };

 /// libCEED Q-function for building quadrature data for a mass operator with a
 /// constant coefficient
@@ -56,8 +56,8 @@ CEED_QFUNCTION(f_build_mass_const)(void *ctx, CeedInt Q,
 }

 /// libCEED Q-function for building quadrature data for a mass operator with a
-/// grid function coefficient
-CEED_QFUNCTION(f_build_mass_grid)(void *ctx, CeedInt Q,
+/// coefficient evaluated at quadrature points.
+CEED_QFUNCTION(f_build_mass_quad)(void *ctx, CeedInt Q,
                                  const CeedScalar *const *in,
                                  CeedScalar *const *out)
 {
@@ -102,11 +102,168 @@ CEED_QFUNCTION(f_apply_mass)(void *ctx, CeedInt Q,
                             const CeedScalar *const *in,
                             CeedScalar *const *out)
 {
+   BuildContext *bc = (BuildContext *)ctx;
   const CeedScalar *u = in[0], *w = in[1];
   CeedScalar *v = out[0];
-   for (CeedInt i=0; i<Q; i++)
+   switch (bc->vdim)
   {
-      v[i] = w[i] * u[i];
+      case 1:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            v[i] = w[i] * u[i];
+         }
+         break;
+      case 2:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            const CeedScalar W = w[i];
+            for (CeedInt c = 0; c < 2; c++)
+            {
+               v[i+c*Q] = W * u[i+c*Q];
+            }
+         }
+         break;
+      case 3:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            const CeedScalar W = w[i];
+            for (CeedInt c = 0; c < 3; c++)
+            {
+               v[i+c*Q] = W * u[i+c*Q];
+            }
+         }
+         break;
+   }
+   return 0;
+}
+
+/// libCEED Q-function for applying a diff operator
+CEED_QFUNCTION(f_apply_mass_mf_const)(void *ctx, CeedInt Q,
+                                      const CeedScalar *const *in, CeedScalar *const *out)
+{
+   BuildContext *bc = (BuildContext*)ctx;
+   const CeedScalar coeff = bc->coeff;
+   const CeedScalar *u = in[0], *J = in[1], *qw = in[2];
+   CeedScalar *v = out[0];
+   switch (10 * bc->dim + bc->vdim)
+   {
+      case 11:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            const CeedScalar rho = coeff * qw[i] / J[i];
+            v[i] = rho * u[i];
+         }
+         break;
+      case 21:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            const CeedScalar rho = coeff * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
+            v[i] = rho * u[i];
+         }
+         break;
+      case 22:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            // 0 2
+            // 1 3
+            const CeedScalar rho = coeff * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
+            for (CeedInt c = 0; c < 2; c++)
+            {
+               v[i+c*Q] = rho * u[i+c*Q];
+            }
+         }
+         break;
+      case 31:
+         for (CeedInt i = 0; i < Q; i++)
+         {
+            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
+                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
+                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * coeff * qw[i];
+            v[i] = rho * u[i];
+         }
+         break;
+      case 33:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            // 0 3 6
+            // 1 4 7
+            // 2 5 8
+            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
+                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
+                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * coeff * qw[i];
+            for (CeedInt c = 0; c < 3; c++)
+            {
+               v[i+c*Q] = rho * u[i+c*Q];
+            }
+         }
+         break;
+   }
+   return 0;
+}
+
+CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
+                                     const CeedScalar *const *in, CeedScalar *const *out)
+{
+   BuildContext *bc = (BuildContext*)ctx;
+   const CeedScalar *c = in[0], *u = in[1], *J = in[2], *qw = in[3];
+   CeedScalar *v = out[0];
+   switch (10 * bc->dim + bc->vdim)
+   {
+      case 11:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            const CeedScalar rho = c[i] * J[i] * qw[i];
+            v[i] = rho * u[i];
+         }
+         break;
+      case 21:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            // 0 2
+            // 1 3
+            const CeedScalar rho = c[i] * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
+            v[i] = rho * u[i];
+         }
+         break;
+      case 22:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            // 0 2
+            // 1 3
+            const CeedScalar rho = c[i] * (J[i+Q*0]*J[i+Q*3] - J[i+Q*1]*J[i+Q*2]) * qw[i];
+            for (CeedInt c = 0; c < 2; c++)
+            {
+               v[i+c*Q] = rho * u[i+c*Q];
+            }
+         }
+         break;
+      case 31:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            // 0 3 6
+            // 1 4 7
+            // 2 5 8
+            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
+                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
+                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * c[i] * qw[i];
+            v[i] = rho * u[i];
+         }
+         break;
+      case 33:
+         for (CeedInt i=0; i<Q; i++)
+         {
+            // 0 3 6
+            // 1 4 7
+            // 2 5 8
+            const CeedScalar rho = (J[i+Q*0]*(J[i+Q*4]*J[i+Q*8] - J[i+Q*5]*J[i+Q*7]) -
+                                    J[i+Q*1]*(J[i+Q*3]*J[i+Q*8] - J[i+Q*5]*J[i+Q*6]) +
+                                    J[i+Q*2]*(J[i+Q*3]*J[i+Q*7] - J[i+Q*4]*J[i+Q*6])) * c[i] * qw[i];
+            for (CeedInt c = 0; c < 3; c++)
+            {
+               v[i+c*Q] = rho * u[i+c*Q];
+            }
+         }
+         break;
   }
   return 0;
 }
@@ -14,7 +14,6 @@

 #include "ceed.hpp"

-#ifdef MFEM_USE_CEED
 #include "../fespace.hpp"

 namespace mfem
@@ -25,8 +24,10 @@ void CeedPAMassAssemble(const FiniteElementSpace &fes,
                        const mfem::IntegrationRule &ir,
                        CeedData& ceedData);

+void CeedMFMassAssemble(const FiniteElementSpace &fes,
+                        const mfem::IntegrationRule &irm,
+                        CeedData& ceedData);
+
 }

-#endif // MFEM_USE_CEED
-
 #endif // MFEM_LIBCEED_MASS_HPP
@@ -131,8 +131,8 @@ void NonlinearForm::Mult(const Vector &x, Vector &y) const
   const Vector &px = Prolongate(x);
   if (P) { aux2.SetSize(P->Height()); }

-   // If we are in parallel, ParNonLinearForm::Mult uses the aux2 vector.
-   // In serial, place the result directly in y.
+   // If we are in parallel, ParNonLinearForm::Mult uses the aux2 vector. In
+   // serial, place the result directly in y.
   Vector &py = P ? aux2 : y;

   if (ext)
@@ -456,11 +456,12 @@ void BlockNonlinearForm::SetSpaces(Array<FiniteElementSpace *> &f)
      for (int j=0; j<Grads.NumCols(); ++j)
      {
         delete Grads(i,j);
+         delete cGrads(i,j);
      }
   }
-   for (int i = 0; i < ess_vdofs.Size(); ++i)
+   for (int i = 0; i < ess_tdofs.Size(); ++i)
   {
-      delete ess_vdofs[i];
+      delete ess_tdofs[i];
   }

   height = 0;
@@ -486,10 +487,33 @@ void BlockNonlinearForm::SetSpaces(Array<FiniteElementSpace *> &f)
   Grads.SetSize(fes.Size(), fes.Size());
   Grads = NULL;

-   ess_vdofs.SetSize(fes.Size());
+   cGrads.SetSize(fes.Size(), fes.Size());
+   cGrads = NULL;
+
+   P.SetSize(fes.Size());
+   cP.SetSize(fes.Size());
+   ess_tdofs.SetSize(fes.Size());
   for (int s = 0; s < fes.Size(); ++s)
   {
-      ess_vdofs[s] = new Array<int>;
+      // Retrieve prolongation matrix for each FE space
+      P[s] = fes[s]->GetProlongationMatrix();
+      cP[s] = dynamic_cast<const SparseMatrix *>(P[s]);
+
+      // If the P Operator exists and its type is not SparseMatrix, this
+      // indicates the Operator is part of parallel run.
+      if (P[s] && !cP[s])
+      {
+         is_serial = false;
+      }
+
+      // If the P Operator exists and its type is SparseMatrix, this indicates
+      // the Operator is serial but needs prolongation on assembly.
+      if (cP[s])
+      {
+         needs_prolongation = true;
+      }
+
+      ess_tdofs[s] = new Array<int>;
   }
 }

@@ -506,45 +530,18 @@ void BlockNonlinearForm::AddBdrFaceIntegrator(BlockNonlinearFormIntegrator *nfi,
   bfnfi_marker.Append(&bdr_attr_marker);
 }

-void BlockNonlinearForm::SetEssentialBC(const
-                                        Array<Array<int> *>&bdr_attr_is_ess,
-                                        Array<Vector *> &rhs)
+void BlockNonlinearForm::SetEssentialBC(
+   const Array<Array<int> *> &bdr_attr_is_ess, Array<Vector *> &rhs)
 {
-   int i, j, vsize, nv;
-
-   for (int s=0; s<fes.Size(); ++s)
+   for (int s = 0; s < fes.Size(); ++s)
   {
-      // First, set u variables
-      vsize = fes[s]->GetVSize();
-      Array<int> vdof_marker(vsize);
+      ess_tdofs[s]->SetSize(ess_tdofs.Size());

-      // virtual call, works in parallel too
-      fes[s]->GetEssentialVDofs(*(bdr_attr_is_ess[s]), vdof_marker);
-      nv = 0;
-      for (i = 0; i < vsize; ++i)
-      {
-         if (vdof_marker[i])
-         {
-            nv++;
-         }
-      }
-
-      ess_vdofs[s]->SetSize(nv);
-
-      for (i = j = 0; i < vsize; ++i)
-      {
-         if (vdof_marker[i])
-         {
-            (*ess_vdofs[s])[j++] = i;
-         }
-      }
+      fes[s]->GetEssentialTrueDofs(*bdr_attr_is_ess[s], *ess_tdofs[s]);

      if (rhs[s])
      {
-         for (i = 0; i < nv; ++i)
-         {
-            (*rhs[s])[(*ess_vdofs[s])[i]] = 0.0;
-         }
+         rhs[s]->SetSubVector(*ess_tdofs[s], 0.0);
      }
   }
 }
@@ -757,18 +754,52 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
      delete vdofs[s];
      delete el_y[s];
      delete el_x[s];
-      by.GetBlock(s).SetSubVector(*ess_vdofs[s], 0.0);
   }
 }

+const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
+{
+   MFEM_VERIFY(bx.Size() == Width(), "invalid input BlockVector size");
+
+   if (needs_prolongation)
+   {
+      aux1.Update(block_offsets);
+      for (int s = 0; s < fes.Size(); s++)
+      {
+         P[s]->Mult(bx.GetBlock(s), aux1.GetBlock(s));
+      }
+      return aux1;
+   }
+   return bx;
+}
+
 void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   xs.Update(x.GetData(), block_offsets);
-   ys.Update(y.GetData(), block_offsets);
+   BlockVector bx(x.GetData(), block_trueOffsets);
+   BlockVector by(y.GetData(), block_trueOffsets);
+
+   const BlockVector &pbx = Prolongate(bx);
+   if (needs_prolongation)
+   {
+      aux2.Update(block_offsets);
+   }
+   BlockVector &pby = needs_prolongation ? aux2 : by;
+
+   xs.Update(pbx.GetData(), block_offsets);
+   ys.Update(pby.GetData(), block_offsets);
   MultBlocked(xs, ys);
+
+   for (int s = 0; s < fes.Size(); s++)
+   {
+      if (cP[s])
+      {
+         cP[s]->MultTranspose(pby.GetBlock(s), by.GetBlock(s));
+      }
+      by.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
+   }
 }

-Operator &BlockNonlinearForm::GetGradientBlocked(const BlockVector &bx) const
+void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
 {
   const int skip_zeros = 0;
   Array<Array<int> *> vdofs(fes.Size());
@@ -780,13 +811,6 @@ Operator &BlockNonlinearForm::GetGradientBlocked(const BlockVector &bx) const
   Array<const FiniteElement *>fe2(fes.Size());
   ElementTransformation * T;

-   if (BlockGrad != NULL)
-   {
-      delete BlockGrad;
-   }
-
-   BlockGrad = new BlockOperator(block_offsets);
-
   for (int i=0; i<fes.Size(); ++i)
   {
      el_x_const[i] = el_x[i] = new Vector();
@@ -951,44 +975,70 @@ Operator &BlockNonlinearForm::GetGradientBlocked(const BlockVector &bx) const
      }
   }

-   for (int s=0; s<fes.Size(); ++s)
-   {
-      for (int i = 0; i < ess_vdofs[s]->Size(); ++i)
-      {
-         for (int j=0; j<fes.Size(); ++j)
-         {
-            if (s==j)
-            {
-               Grads(s,s)->EliminateRowCol((*ess_vdofs[s])[i], Matrix::DIAG_ONE);
-            }
-            else
-            {
-               Grads(s,j)->EliminateRow((*ess_vdofs[s])[i]);
-               Grads(j,s)->EliminateCol((*ess_vdofs[s])[i]);
-            }
-         }
-      }
-   }
-
   for (int i=0; i<fes.Size(); ++i)
   {
      for (int j=0; j<fes.Size(); ++j)
      {
-         BlockGrad->SetBlock(i,j,Grads(i,j));
         delete elmats(i,j);
      }
      delete vdofs2[i];
      delete vdofs[i];
      delete el_x[i];
   }
-
-   return *BlockGrad;
 }

 Operator &BlockNonlinearForm::GetGradient(const Vector &x) const
 {
-   xs.Update(x.GetData(), block_offsets);
-   return GetGradientBlocked(xs);
+   BlockVector bx(x.GetData(), block_trueOffsets);
+   const BlockVector &pbx = Prolongate(bx);
+
+   ComputeGradientBlocked(pbx);
+
+   Array2D<SparseMatrix *> mGrads(fes.Size(), fes.Size());
+   mGrads = Grads;
+   if (needs_prolongation)
+   {
+      for (int s1 = 0; s1 < fes.Size(); ++s1)
+      {
+         for (int s2 = 0; s2 < fes.Size(); ++s2)
+         {
+            delete cGrads(s1, s2);
+            cGrads(s1, s2) = RAP(*cP[s1], *Grads(s1, s2), *cP[s2]);
+            mGrads(s1, s2) = cGrads(s1, s2);
+         }
+      }
+   }
+
+   for (int s = 0; s < fes.Size(); ++s)
+   {
+      for (int i = 0; i < ess_tdofs[s]->Size(); ++i)
+      {
+         for (int j = 0; j < fes.Size(); ++j)
+         {
+            if (s == j)
+            {
+               mGrads(s, s)->EliminateRowCol((*ess_tdofs[s])[i],
+                                             Matrix::DIAG_ONE);
+            }
+            else
+            {
+               mGrads(s, j)->EliminateRow((*ess_tdofs[s])[i]);
+               mGrads(j, s)->EliminateCol((*ess_tdofs[s])[i]);
+            }
+         }
+      }
+   }
+
+   delete BlockGrad;
+   BlockGrad = new BlockOperator(block_trueOffsets);
+   for (int i = 0; i < fes.Size(); ++i)
+   {
+      for (int j = 0; j < fes.Size(); ++j)
+      {
+         BlockGrad->SetBlock(i, j, mGrads(i, j));
+      }
+   }
+   return *BlockGrad;
 }

 BlockNonlinearForm::~BlockNonlinearForm()
@@ -999,8 +1049,9 @@ BlockNonlinearForm::~BlockNonlinearForm()
      for (int j=0; j<fes.Size(); ++j)
      {
         delete Grads(i,j);
+         delete cGrads(i,j);
      }
-      delete ess_vdofs[i];
+      delete ess_tdofs[i];
   }

   for (int i = 0; i < dnfi.Size(); ++i)
@@ -199,24 +199,41 @@ protected:
       GridFunction-like block-vector data (e.g. in parallel). */
   mutable BlockVector xs, ys;

-   mutable Array2D<SparseMatrix*> Grads;
+   mutable Array2D<SparseMatrix*> Grads, cGrads;
   mutable BlockOperator *BlockGrad;

   // A list of the offsets
   Array<int> block_offsets;
   Array<int> block_trueOffsets;

-   // Essential vdofs: one list of vdofs for each space in 'fes'
-   Array<Array<int> *> ess_vdofs;
+   // Array of Arrays of tdofs for each space in 'fes'
+   Array<Array<int> *> ess_tdofs;
+
+   /// Array of pointers to the prolongation matrix of fes, may be NULL
+   Array<const Operator *> P;
+
+   /// Array of results of dynamic-casting P to SparseMatrix pointer
+   Array<const SparseMatrix *> cP;
+
+   /// Indicator if the Operator is part of a parallel run
+   bool is_serial = true;
+
+   /// Indicator if the Operator needs prolongation on assembly
+   bool needs_prolongation = false;
+
+   mutable BlockVector aux1, aux2;
+
+   const BlockVector &Prolongate(const BlockVector &bx) const;

   /// Specialized version of GetEnergy() for BlockVectors
   double GetEnergyBlocked(const BlockVector &bx) const;

   /// Specialized version of Mult() for BlockVector%s
+   /// Block L-Vector to Block L-Vector
   void MultBlocked(const BlockVector &bx, BlockVector &by) const;

   /// Specialized version of GetGradient() for BlockVector
-   Operator &GetGradientBlocked(const BlockVector &bx) const;
+   void ComputeGradientBlocked(const BlockVector &bx) const;

 public:
   /// Construct an empty BlockNonlinearForm. Initialize with SetSpaces().
@@ -261,8 +278,12 @@ public:

   virtual double GetEnergy(const Vector &x) const;

+   /// Method is only called in serial, the parallel version calls MultBlocked
+   /// directly.
   virtual void Mult(const Vector &x, Vector &y) const;

+   /// Method is only called in serial, the parallel version calls
+   /// GetGradientBlocked directly.
   virtual Operator &GetGradient(const Vector &x) const;

   /// Destructor.
@@ -76,7 +76,7 @@ public:
   void KeepNbrBlock(bool knb = true) { keep_nbr_block = knb; }

   /** @brief Set the operator type id for the parallel matrix/operator when
-       using AssemblyLevel::FULL. */
+       using AssemblyLevel::LEGACYFULL. */
   /** If using static condensation or hybridization, call this method *after*
       enabling it. */
   void SetOperatorType(Operator::Type tid)
@@ -1999,13 +1999,13 @@ int ParFiniteElementSpace
      for (int entity = 0; entity <= 2; entity++)
      {
         const NCMesh::NCList &list = pncmesh->GetNCList(entity);
-         if (!list.masters.size()) { continue; }
+         if (!list.masters.Size()) { continue; }

         IsoparametricTransformation T;
         DenseMatrix I;

         // process masters that we own or that affect our edges/faces
-         for (unsigned mi = 0; mi < list.masters.size(); mi++)
+         for (int mi = 0; mi < list.masters.Size(); mi++)
         {
            const NCMesh::Master &mf = list.masters[mi];

@@ -2036,7 +2036,7 @@ int ParFiniteElementSpace
               GetEntityDofs(entity, sf.index, slave_dofs, mf.Geom());
               if (!slave_dofs.Size()) { continue; }

-               sf.OrientedPointMatrix(T.GetPointMat());
+               list.OrientedPointMatrix(sf, T.GetPointMat());
               fe->GetLocalInterpolation(T, I);

               // make each slave DOF dependent on all master DOFs
@@ -2064,12 +2064,12 @@ int ParFiniteElementSpace
      {
         const NCMesh::NCList &list = pncmesh->GetNCList(entity);

-         std::size_t lsize[3] =
-         { list.conforming.size(), list.masters.size(), list.slaves.size() };
+         int lsize[3] =
+         { list.conforming.Size(), list.masters.Size(), list.slaves.Size() };

         for (int l = 0; l < 3; l++)
         {
-            for (std::size_t i = 0; i < lsize[l]; i++)
+            for (int i = 0; i < lsize[l]; i++)
            {
               const MeshId &id =
                  (l == 0) ? list.conforming[i] :
@@ -1093,6 +1093,9 @@ double L2ZZErrorEstimator(BilinearFormIntegrator &flux_integrator,

   if (xfes->GetNE())
   {
+      MFEM_VERIFY(smooth_flux_fes.GetFE(0) != NULL,
+                  "Could not obtain FE of smooth flux space.");
+
      if (smooth_flux_fes.GetFE(0)->GetRangeType() == FiniteElement::SCALAR)
      {
         VectorMassIntegrator *vmass = new VectorMassIntegrator;
@@ -200,23 +200,34 @@ void ParBlockNonlinearForm::SetEssentialBC(const

   BlockNonlinearForm::SetEssentialBC(bdr_attr_is_ess, nullarray);

-   for (int s=0; s<fes.Size(); ++s)
+   for (int s = 0; s < fes.Size(); ++s)
   {
      if (rhs[s])
      {
-         ParFiniteElementSpace *pfes = ParFESpace(s);
-         for (int i=0; i < ess_vdofs[s]->Size(); ++i)
-         {
-            int tdof = pfes->GetLocalTDofNumber((*(ess_vdofs[s]))[i]);
-            if (tdof >= 0)
-            {
-               (*rhs[s])(tdof) = 0.0;
-            }
-         }
+         rhs[s]->SetSubVector(*ess_tdofs[s], 0.0);
      }
   }
 }

+double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
+{
+   xs_true.Update(x.GetData(), block_trueOffsets);
+   xs.Update(block_offsets);
+
+   for (int s = 0; s < fes.Size(); ++s)
+   {
+      fes[s]->GetProlongationMatrix()->Mult(xs_true.GetBlock(s), xs.GetBlock(s));
+   }
+
+   double enloc = BlockNonlinearForm::GetEnergyBlocked(xs);
+   double englo = 0.0;
+
+   MPI_Allreduce(&enloc, &englo, 1, MPI_DOUBLE, MPI_SUM,
+                 ParFESpace(0)->GetComm());
+
+   return englo;
+}
+
 void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
   xs_true.Update(x.GetData(), block_trueOffsets);
@@ -241,6 +252,8 @@ void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
   {
      fes[s]->GetProlongationMatrix()->MultTranspose(
         ys.GetBlock(s), ys_true.GetBlock(s));
+
+      ys_true.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
   }
 }

@@ -257,8 +270,18 @@ const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
         xs_true.GetBlock(s), xs.GetBlock(s));
   }

-   BlockNonlinearForm::GetGradientBlocked(xs); // (re)assemble Grad with b.c.
+   BlockNonlinearForm::ComputeGradientBlocked(xs); // (re)assemble Grad with b.c.

+   delete BlockGrad;
+   BlockGrad = new BlockOperator(block_offsets);
+
+   for (int i = 0; i < fes.Size(); ++i)
+   {
+      for (int j = 0; j < fes.Size(); ++j)
+      {
+         BlockGrad->SetBlock(i, j, Grads(i, j));
+      }
+   }
   return *BlockGrad;
 }

@@ -314,6 +337,9 @@ BlockOperator & ParBlockNonlinearForm::GetGradient(const Vector &x) const
                                   pfes[s1]->GetDofOffsets(), Grads(s1,s1));
            Ph.ConvertFrom(pfes[s1]->Dof_TrueDof_Matrix());
            phBlockGrad(s1,s1)->MakePtAP(dA, Ph);
+
+            OperatorHandle Ae;
+            Ae.EliminateRowsCols(*phBlockGrad(s1,s1), *ess_tdofs[s1]);
         }
         else
         {
@@ -327,6 +353,9 @@ BlockOperator & ParBlockNonlinearForm::GetGradient(const Vector &x) const
            Ph.ConvertFrom(pfes[s2]->Dof_TrueDof_Matrix());

            phBlockGrad(s1,s2)->MakeRAP(Rh, dA, Ph);
+
+            phBlockGrad(s1,s2)->EliminateRows(*ess_tdofs[s1]);
+            phBlockGrad(s1,s2)->EliminateCols(*ess_tdofs[s2]);
         }

         pBlockGrad->SetBlock(s1, s2, phBlockGrad(s1,s2)->Ptr());
@@ -82,6 +82,9 @@ protected:
   mutable BlockOperator *pBlockGrad;

 public:
+   /// Computes the energy of the system
+   virtual double GetEnergy(const Vector &x) const;
+
   /// Construct an empty ParBlockNonlinearForm. Initialize with SetParSpaces().
   ParBlockNonlinearForm() : pBlockGrad(NULL) { }

@@ -103,6 +106,7 @@ public:
   virtual void SetEssentialBC(const Array<Array<int> *>&bdr_attr_is_ess,
                               Array<Vector *> &rhs);

+   /// Block T-Vector to Block T-Vector
   virtual void Mult(const Vector &x, Vector &y) const;

   /// Return the local block gradient matrix for the given true-dof vector x
@@ -298,7 +298,7 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
         element_nvertices = static_cast<size_t>(mesh.elements[0]->GetNVertices());
      }
      SafeDefineVariable<uint64_t>(io, "connectivity", {}, {}, {nelements, element_nvertices+1});
-      SafeDefineVariable<int32_t>(io, "material", {}, {}, {nelements});
+      SafeDefineVariable<int32_t>(io, "attribute", {}, {}, {nelements});

      // vertices
      SafeDefineVariable<uint32_t>(io,"NumOfVertices", {adios2::LocalValueDim});
@@ -351,7 +351,7 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
                                                           (var_connectivity);

      adios2::Variable<int32_t> var_element_attribute =
-         io.InquireVariable<int32_t>("material");
+         io.InquireVariable<int32_t>("attribute");
      adios2::Variable<int32_t>::Span span_element_attribute = engine.Put<int32_t>
                                                               (var_element_attribute);

@@ -435,7 +435,7 @@ void adios2stream::Print(const Mesh& mesh, const mode print_mode)
         engine.Put<uint64_t>(varConnectivity);

      adios2::Variable<int32_t> varElementAttribute =
-         io.InquireVariable<int32_t>("material");
+         io.InquireVariable<int32_t>("attribute");
      // zero-copy access to adios2 buffer to put non-contiguous to contiguous memory
      adios2::Variable<int32_t>::Span spanElementAttribute =
         engine.Put<int32_t>(varElementAttribute);
@@ -720,7 +720,7 @@ std::string adios2stream::VTKSchema() const noexcept
   vtkSchema += R"(
      </Points>
 	  <CellData>
-        <DataArray Name="material" />
+        <DataArray Name="attribute" />
 	  </CellData>
      <Cells>
        <DataArray Name="connectivity" />
@@ -411,6 +411,8 @@ public:
   inline void operator=(const T &a)
   { array1d = a; }

+   inline Array2D& operator=(const Array2D &a) = default;
+
   /// Make this Array a reference to 'master'
   inline void MakeRef(const Array2D &master)
   { M = master.M; N = master.N; array1d.MakeRef(master.array1d); }
@@ -31,6 +31,7 @@
 #endif

 #ifdef MFEM_USE_RAJA
+#define CAMP_USE_PLATFORM_DEFAULT_STREAM 1
 #include "RAJA/RAJA.hpp"
 #if defined(RAJA_ENABLE_CUDA) && !defined(MFEM_USE_CUDA)
 #error When RAJA is built with CUDA, MFEM_USE_CUDA=YES is required
@@ -11,6 +11,9 @@

 #include "forall.hpp"
 #include "occa.hpp"
+#if defined(MFEM_USE_OCCA) && OCCA_CUDA_ENABLED
+#include <occa/modes/cuda/stream.hpp>
+#endif
 #ifdef MFEM_USE_CEED
 #include "../fem/libceed/ceed.hpp"
 #endif
@@ -43,7 +46,7 @@ CeedRestrMap ceed_restr_map;
 static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 {
   Backend::CEED_CUDA, Backend::OCCA_CUDA, Backend::RAJA_CUDA, Backend::CUDA,
-   Backend::HIP, Backend::DEBUG_DEVICE,
+   Backend::CEED_HIP, Backend::HIP, Backend::DEBUG_DEVICE,
   Backend::OCCA_OMP, Backend::RAJA_OMP, Backend::OMP,
   Backend::CEED_CPU, Backend::OCCA_CPU, Backend::RAJA_CPU, Backend::CPU
 };
@@ -52,7 +55,7 @@ static const Backend::Id backend_list[Backend::NUM_BACKENDS] =
 static const char *backend_name[Backend::NUM_BACKENDS] =
 {
   "ceed-cuda", "occa-cuda", "raja-cuda", "cuda",
-   "hip", "debug",
+   "ceed-hip", "hip", "debug",
   "occa-omp", "raja-omp", "omp",
   "ceed-cpu", "occa-cpu", "raja-cpu", "cpu"
 };
@@ -163,10 +166,12 @@ Device::~Device()
      {
         CeedBasisDestroy(&entry.second);
      }
+      internal::ceed_basis_map.clear();
      for (auto entry : internal::ceed_restr_map)
      {
         CeedElemRestrictionDestroy(&entry.second);
      }
+      internal::ceed_restr_map.clear();
      // Destroy Ceed context
      CeedDestroy(&internal::ceed);
 #endif
@@ -233,6 +238,10 @@ void Device::Configure(const std::string &device, const int dev)
   {
      Get().MarkBackend(Backend::CUDA);
   }
+   if (Allows(Backend::CEED_HIP))
+   {
+      Get().MarkBackend(Backend::HIP);
+   }

   // Perform setup.
   Get().Setup(dev);
@@ -404,6 +413,10 @@ static void OccaDeviceSetup(const int dev)
 #if OCCA_CUDA_ENABLED
      std::string mode("mode: 'CUDA', device_id : ");
      internal::occaDevice.setup(mode.append(1,'0'+dev));
+      auto def_stream = new occa::cuda::stream(
+         internal::occaDevice.getModeDevice(),
+         occa::properties(), (CUstream)0);
+      internal::occaDevice.setStream(def_stream);
 #else
      MFEM_ABORT("the OCCA CUDA backend requires OCCA built with CUDA!");
 #endif
@@ -449,7 +462,8 @@ static void CeedDeviceSetup(const char* ceed_spec)
   CeedInit(ceed_spec, &internal::ceed);
   const char *ceed_backend;
   CeedGetResource(internal::ceed, &ceed_backend);
-   if (strcmp(ceed_spec, ceed_backend) && strcmp(ceed_spec, "/cpu/self"))
+   if (strcmp(ceed_spec, ceed_backend) && strcmp(ceed_spec, "/cpu/self") &&
+       strcmp(ceed_spec, "/gpu/hip"))
   {
      mfem::out << std::endl << "WARNING!!!\n"
                "libCEED is not using the requested backend!!!\n"
@@ -518,6 +532,17 @@ void Device::Setup(const int device)
         CeedDeviceSetup(device_option);
      }
   }
+   if (Allows(Backend::CEED_HIP))
+   {
+      if (!device_option)
+      {
+         CeedDeviceSetup("/gpu/hip");
+      }
+      else
+      {
+         CeedDeviceSetup(device_option);
+      }
+   }
   if (Allows(Backend::DEBUG_DEVICE)) { ngpu = 1; }
 }

@@ -59,14 +59,17 @@ struct Backend
      CEED_CPU  = 1 << 10,
      /** @brief [device] CEED CUDA backend working together with the CUDA
          backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_CUDA = YES.
-          NOTE: The current default libCEED GPU backend is non-deterministic! */
+          NOTE: The current default libCEED CUDA backend is non-deterministic! */
      CEED_CUDA = 1 << 11,
+      /** @brief [device] CEED HIP backend working together with the HIP
+          backend. Enabled when MFEM_USE_CEED = YES and MFEM_USE_HIP = YES. */
+      CEED_HIP = 1 << 12,
      /** @brief [device] Debug backend: host memory is READ/WRITE protected
          while a device is in use. It allows to test the "device" code-path
          (using separate host/device memory pools and host <-> device
          transfers) without any GPU hardware. As 'DEBUG' is sometimes used
          as a macro, `_DEVICE` has been added to avoid conflicts. */
-      DEBUG_DEVICE = 1 << 12
+      DEBUG_DEVICE = 1 << 13
   };

   /** @brief Additional useful constants. For example, the *_MASK constants can
@@ -74,18 +77,18 @@ struct Backend
   enum
   {
      /// Number of backends: from (1 << 0) to (1 << (NUM_BACKENDS-1)).
-      NUM_BACKENDS = 13,
+      NUM_BACKENDS = 14,

      /// Biwise-OR of all CPU backends
      CPU_MASK = CPU | RAJA_CPU | OCCA_CPU | CEED_CPU,
      /// Biwise-OR of all CUDA backends
      CUDA_MASK = CUDA | RAJA_CUDA | OCCA_CUDA | CEED_CUDA,
      /// Biwise-OR of all HIP backends
-      HIP_MASK = HIP,
+      HIP_MASK = HIP | CEED_HIP,
      /// Biwise-OR of all OpenMP backends
      OMP_MASK = OMP | RAJA_OMP | OCCA_OMP,
      /// Bitwise-OR of all CEED backends
-      CEED_MASK = CEED_CPU | CEED_CUDA,
+      CEED_MASK = CEED_CPU | CEED_CUDA | CEED_HIP,
      /// Biwise-OR of all device backends
      DEVICE_MASK = CUDA_MASK | HIP_MASK | DEBUG_DEVICE,

@@ -15,6 +15,9 @@
 #include "../config/config.hpp"
 #include <iomanip>
 #include <sstream>
+#ifdef MFEM_USE_HIP
+#include <hip/hip_runtime.h>
+#endif

 namespace mfem
 {
@@ -144,6 +147,22 @@ void mfem_warning(const char *msg = NULL);
   "invalid index " #i << " = " << (i) << \
   ", valid range is [" << (imin) << ',' << (imax) << ')')

+
+// Additional abort functions for HIP
+#if defined(MFEM_USE_HIP)
+template<typename T>
+__host__ void abort_msg(T & msg)
+{
+   MFEM_ABORT(msg);
+}
+
+template<typename T>
+__device__ void abort_msg(T & msg)
+{
+   abort();
+}
+#endif
+
 // Abort inside a device kernel
 #if defined(__CUDA_ARCH__)
 #define MFEM_ABORT_KERNEL(msg) \
@@ -151,6 +170,11 @@ void mfem_warning(const char *msg = NULL);
      printf(msg);             \
      asm("trap;");            \
   }
+#elif defined(MFEM_USE_HIP)
+#define MFEM_ABORT_KERNEL(msg) \
+   {                           \
+      abort_msg(msg);          \
+   }
 #else
 #define MFEM_ABORT_KERNEL(msg) MFEM_ABORT(msg)
 #endif
@@ -115,6 +115,12 @@ const char *GetConfigStr()
 #ifdef MFEM_USE_STRUMPACK
      "MFEM_USE_STRUMPACK\n"
 #endif
+#ifdef MFEM_USE_GINKGO
+      "MFEM_USE_GINKGO\n"
+#endif
+#ifdef MFEM_USE_AMGX
+      "MFEM_USE_AMGX\n"
+#endif
 #ifdef MFEM_USE_GNUTLS
      "MFEM_USE_GNUTLS\n"
 #endif
@@ -136,6 +142,9 @@ const char *GetConfigStr()
 #ifdef MFEM_USE_PUMI
      "MFEM_USE_PUMI\n"
 #endif
+#ifdef MFEM_USE_HIOP
+      "MFEM_USE_HIOP\n"
+#endif
 #ifdef MFEM_USE_CUDA
      "MFEM_USE_CUDA\n"
 #endif
@@ -57,12 +57,6 @@ if (MFEM_USE_MPI)
  list(APPEND HDRS
    hypre.hpp
    hypre_parcsr.hpp)
-  if (MFEM_USE_AMGX)
-  list(APPEND SRCS
-  amgxsolver.cpp)
-  list(APPEND HDRS
-  amgxsolver.hpp)
-  endif()
  if (MFEM_USE_PETSC)
    list(APPEND SRCS
      petsc.cpp)
@@ -77,11 +71,6 @@ if (MFEM_USE_MPI)
  endif()
 endif()

-if (MFEM_USE_GINKGO)
-  list(APPEND SRCS ginkgo.cpp)
-  list(APPEND HDRS ginkgo.hpp)
-endif()
-
 if (MFEM_USE_SUNDIALS)
  list(APPEND SRCS sundials.cpp)
  list(APPEND HDRS sundials.hpp)
@@ -99,11 +88,26 @@ if (MFEM_USE_STRUMPACK)
  list(APPEND HDRS strumpack.hpp)
 endif()

+if (MFEM_USE_GINKGO)
+  list(APPEND SRCS ginkgo.cpp)
+  list(APPEND HDRS ginkgo.hpp)
+endif()
+
+if (MFEM_USE_AMGX)
+  list(APPEND SRCS amgxsolver.cpp)
+  list(APPEND HDRS amgxsolver.hpp)
+endif()
+
 if (MFEM_USE_HIOP)
  list(APPEND SRCS hiop.cpp)
  list(APPEND HDRS hiop.hpp)
 endif()

+if (MFEM_USE_MKL_CPARDISO)
+  list(APPEND SRCS cpardiso.cpp)
+  list(APPEND HDRS cpardiso.hpp)
+endif()
+
 convert_filenames_to_full_paths(SRCS)
 convert_filenames_to_full_paths(HDRS)

@@ -9,10 +9,15 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-//Reference
-//Pi-Yueh Chuang, & Lorena A. Barba (2017).
-//AmgXWrapper: An interface between PETSc and the NVIDIA AmgX library. J.
-//Open Source Software, 2(16):280, doi:10.21105/joss.00280
+// Implementation of the MFEM wrapper for Nvidia's multigrid library, AmgX
+//
+// This work is partially based on:
+//
+//    Pi-Yueh Chuang and Lorena A. Barba (2017).
+//    AmgXWrapper: An interface between PETSc and the NVIDIA AmgX library.
+//    J. Open Source Software, 2(16):280, doi:10.21105/joss.00280
+//
+// See https://github.com/barbagroup/AmgXWrapper.

 #include "../config/config.hpp"
 #include "amgxsolver.hpp"
@@ -104,19 +109,18 @@ void AmgXSolver::InitExclusiveGPU(const MPI_Comm &comm)
      mfem_error("This AmgXSolver instance has been initialized on this process.");
   }

-   //Note that every MPI rank may talk to a GPU
+   // Note that every MPI rank may talk to a GPU
   mpi_gpu_mode = "mpi-gpu-exclusive";
   gpuProc = 0;

-   //Increment number of AmgX instances
+   // Increment number of AmgX instances
   count++;

   MPI_Comm_dup(comm, &gpuWorld);
   MPI_Comm_size(gpuWorld, &gpuWorldSize);
   MPI_Comm_rank(gpuWorld, &myGpuWorldRank);

-   //Each rank will only see 1 device
-   //call it device 0
+   // Each rank will only see 1 device call it device 0
   nDevs = 1, devID = 0;

   InitAmgX();
@@ -124,8 +128,8 @@ void AmgXSolver::InitExclusiveGPU(const MPI_Comm &comm)
   isInitialized = true;
 }

-// Intialize for MPI ranks  > GPUs, all devices are visible
-// to all of the MPI ranks
+// Initialize for MPI ranks > GPUs, all devices are visible to all of the MPI
+// ranks
 void AmgXSolver::InitMPITeams(const MPI_Comm &comm,
                              const int nDevs)
 {
@@ -137,7 +141,7 @@ void AmgXSolver::InitMPITeams(const MPI_Comm &comm,

   mpi_gpu_mode = "mpi-teams";

-   //Increment number of AmgX instances
+   // Increment number of AmgX instances
   count++;

   // Get the name of this node
@@ -221,7 +225,7 @@ void AmgXSolver::DefaultParameters(const AMGX_MODE amgxMode_,
                    "     \"interpolator\": \"D2\", \n"
                    "     \"max_row_sum\" : 0.9, \n"
                    "     \"strength_threshold\" : 0.25, \n"
-                    "     \"max_iters\": 1, \n"
+                    "     \"max_iters\": 2, \n"
                    "     \"scope\": \"amg\", \n"
                    "     \"max_levels\": 100, \n"
                    "     \"cycle\": \"V\", \n"
@@ -267,6 +271,12 @@ void AmgXSolver::InitAmgX()
      AMGX_SAFE_CALL(AMGX_initialize_plugins());

      AMGX_SAFE_CALL(AMGX_install_signal_handler());
+
+      AMGX_SAFE_CALL(AMGX_register_print_callback(
+                        [](const char *msg, int length)->void
+      {
+         int irank; MPI_Comm_rank(MPI_COMM_WORLD, &irank);
+         if (irank == 0) { mfem::out<<msg;} }));
   }

   MFEM_VERIFY(configSrc != CONFIG_SRC::UNDEFINED,
@@ -392,7 +402,7 @@ void AmgXSolver::SetDeviceIDs(const int nDevs)
 void AmgXSolver::GatherArray(const Array<double> &inArr, Array<double> &outArr,
                             const int mpiTeamSz, const MPI_Comm &mpiTeamComm) const
 {
-   //Calculate number of elements to be collected from each process
+   // Calculate number of elements to be collected from each process
   Array<int> Apart(mpiTeamSz);
   int locAsz = inArr.Size();
   MPI_Gather(&locAsz, 1, MPI_INT,
@@ -400,7 +410,7 @@ void AmgXSolver::GatherArray(const Array<double> &inArr, Array<double> &outArr,

   MPI_Barrier(mpiTeamComm);

-   //Determine stride for process (to be used by root)
+   // Determine stride for process (to be used by root)
   Array<int> Adisp(mpiTeamSz);
   int myid; MPI_Comm_rank(mpiTeamComm, &myid);
   if (myid == 0)
@@ -420,7 +430,7 @@ void AmgXSolver::GatherArray(const Array<double> &inArr, Array<double> &outArr,
 void AmgXSolver::GatherArray(const Vector &inArr, Vector &outArr,
                             const int mpiTeamSz, const MPI_Comm &mpiTeamComm) const
 {
-   //Calculate number of elements to be collected from each process
+   // Calculate number of elements to be collected from each process
   Array<int> Apart(mpiTeamSz);
   int locAsz = inArr.Size();
   MPI_Gather(&locAsz, 1, MPI_INT,
@@ -428,7 +438,7 @@ void AmgXSolver::GatherArray(const Vector &inArr, Vector &outArr,

   MPI_Barrier(mpiTeamComm);

-   //Determine stride for process (to be used by root)
+   // Determine stride for process (to be used by root)
   Array<int> Adisp(mpiTeamSz);
   int myid; MPI_Comm_rank(mpiTeamComm, &myid);
   if (myid == 0)
@@ -448,7 +458,7 @@ void AmgXSolver::GatherArray(const Vector &inArr, Vector &outArr,
 void AmgXSolver::GatherArray(const Array<int> &inArr, Array<int> &outArr,
                             const int mpiTeamSz, const MPI_Comm &mpiTeamComm) const
 {
-   //Calculate number of elements to be collected from each process
+   // Calculate number of elements to be collected from each process
   Array<int> Apart(mpiTeamSz);
   int locAsz = inArr.Size();
   MPI_Gather(&locAsz, 1, MPI_INT,
@@ -456,7 +466,7 @@ void AmgXSolver::GatherArray(const Array<int> &inArr, Array<int> &outArr,

   MPI_Barrier(mpiTeamComm);

-   //Determine stride for process (to be used by root)
+   // Determine stride for process (to be used by root)
   Array<int> Adisp(mpiTeamSz);
   int myid; MPI_Comm_rank(mpiTeamComm, &myid);
   if (myid == 0)
@@ -478,7 +488,7 @@ void AmgXSolver::GatherArray(const Array<int64_t> &inArr,
                             Array<int64_t> &outArr,
                             const int mpiTeamSz, const MPI_Comm &mpiTeamComm) const
 {
-   //Calculate number of elements to be collected from each process
+   // Calculate number of elements to be collected from each process
   Array<int> Apart(mpiTeamSz);
   int locAsz = inArr.Size();
   MPI_Gather(&locAsz, 1, MPI_INT,
@@ -486,7 +496,7 @@ void AmgXSolver::GatherArray(const Array<int64_t> &inArr,

   MPI_Barrier(mpiTeamComm);

-   //Determine stride for process
+   // Determine stride for process
   Array<int> Adisp(mpiTeamSz);
   int myid; MPI_Comm_rank(mpiTeamComm, &myid);
   if (myid == 0)
@@ -509,14 +519,14 @@ void AmgXSolver::GatherArray(const Vector &inArr, Vector &outArr,
                             const int mpiTeamSz, const MPI_Comm &mpiTeamComm,
                             Array<int> &Apart, Array<int> &Adisp) const
 {
-   //Calculate number of elements to be collected from each process
+   // Calculate number of elements to be collected from each process
   int locAsz = inArr.Size();
   MPI_Allgather(&locAsz, 1, MPI_INT,
                 Apart.HostWrite(),1, MPI_INT, mpiTeamComm);

   MPI_Barrier(mpiTeamComm);

-   //Determine stride for process
+   // Determine stride for process
   Adisp[0] = 0;
   for (int i=1; i<mpiTeamSz; ++i)
   {
@@ -565,7 +575,7 @@ void AmgXSolver::SetMatrix(const SparseMatrix &in_A, const bool update_mat)
 #ifdef MFEM_USE_MPI
 void AmgXSolver::SetMatrix(const HypreParMatrix &A, const bool update_mat)
 {
-   //Require hypre >= 2.16.
+   // Require hypre >= 2.16.
 #if MFEM_HYPRE_VERSION < 21600
   mfem_error("Hypre version 2.16+ is required when using AmgX \n");
 #endif
@@ -578,14 +588,14 @@ void AmgXSolver::SetMatrix(const HypreParMatrix &A, const bool update_mat)
   Array<double> loc_A(A_csr->data, (int)A_csr->num_nonzeros);
   const Array<int> loc_I(A_csr->i, (int)A_csr->num_rows+1);

-   //Column index must be int64_t so we must promote here
+   // Column index must be int64_t so we must promote here
   Array<int64_t> loc_J((int)A_csr->num_nonzeros);
   for (int i=0; i<A_csr->num_nonzeros; ++i)
   {
      loc_J[i] = A_csr->big_j[i];
   }

-   //Asumes one GPU per MPI rank
+   // Assumes one GPU per MPI rank
   if (mpi_gpu_mode=="mpi-gpu-exclusive")
   {
      return SetMatrixMPIGPUExclusive(A, loc_A, loc_I, loc_J, update_mat);
@@ -606,7 +616,7 @@ void AmgXSolver::SetMatrixMPIGPUExclusive(const HypreParMatrix &A,
                                          const Array<int64_t> &loc_J,
                                          const bool update_mat)
 {
-   //Create a vector of offsets describing matrix row partitions
+   // Create a vector of offsets describing matrix row partitions
   Array<int64_t> rowPart(gpuWorldSize+1); rowPart = 0.0;

   int64_t myStart = A.GetRowStarts()[0];
@@ -655,16 +665,16 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,
                                   const Array<int64_t> &loc_J,
                                   const bool update_mat)
 {
-   // The following arrays hold the consolidated
-   // diagonal + off diagonal matrix data
+   // The following arrays hold the consolidated diagonal + off-diagonal matrix
+   // data
   Array<int> all_I;
   Array<int64_t> all_J;
   Array<double> all_A;

-   //Determine array sizes
+   // Determine array sizes
   int J_allsz(0), all_NNZ(0), nDevRows(0);
   const int loc_row_len = std::abs(A.RowPart()[1] -
-                                    A.RowPart()[0]); //end of row partition
+                                    A.RowPart()[0]); // end of row partition
   const int loc_Jz_sz = loc_J.Size();
   const int loc_A_sz = loc_A.Size();

@@ -692,13 +702,13 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,

   if (myDevWorldRank == 0)
   {
-      // A fix up step is needed for the array holding row data
-      // to remove extra zeros when consolidating team data.
+      // A fix up step is needed for the array holding row data to remove extra
+      // zeros when consolidating team data.
      Array<int> z_ind(devWorldSize+1);
      int iter = 1;
      while (iter < devWorldSize-1)
      {
-         //Determine the indices of zeros in global all_I array
+         // Determine the indices of zeros in global all_I array
         int counter = 0;
         z_ind[counter] = counter;
         counter++;
@@ -711,15 +721,16 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,
            }
         }
         z_ind[devWorldSize] = all_I.Size()-1;
-         //End of determining indices of zeros in global all_I Array
+         // End of determining indices of zeros in global all_I Array

-         //Bump all_I
+         // Bump all_I
         for (int idx=z_ind[1]+1; idx < z_ind[2]; idx++)
         {
            all_I[idx] = all_I[idx-1] + (all_I[idx+1] - all_I[idx]);
         }

-         //Shift array after bump to remove uncesssary values in middle of array
+         // Shift array after bump to remove unnecessary values in middle of
+         // array
         for (int idx=z_ind[2]; idx < all_I.Size()-1; ++idx)
         {
            all_I[idx] = all_I[idx+1];
@@ -728,7 +739,7 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,
      }

      // LAST TIME THROUGH ARRAY
-      //Determine the indices of zeros in global row_ptr array
+      // Determine the indices of zeros in global row_ptr array
      int counter = 0;
      z_ind[counter] = counter;
      counter++;
@@ -742,8 +753,8 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,
      }

      z_ind[devWorldSize] = all_I.Size()-1;
-      //End of determining indices of zeros in global all_I Array
-      //BUMP all_I one last time
+      // End of determining indices of zeros in global all_I Array BUMP all_I
+      // one last time
      for (int idx=z_ind[1]+1; idx < all_I.Size()-1; idx++)
      {
         all_I[idx] = all_I[idx-1] + (all_I[idx+1] - all_I[idx]);
@@ -752,8 +763,8 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,
      local_rows = nDevRows;
   }

-   //Create row partition
-   mat_local_rows = local_rows; //class copy
+   // Create row partition
+   mat_local_rows = local_rows; // class copy
   Array<int64_t> rowPart;
   if (gpuProc == 0)
   {
@@ -764,13 +775,13 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,
                    gpuWorld);
      MPI_Barrier(gpuWorld);

-      //Fixup step
+      // Fixup step
      for (int i=1; i<rowPart.Size(); ++i)
      {
         rowPart[i] += rowPart[i-1];
      }

-      //upload A matrix to AmgX
+      // Upload A matrix to AmgX
      MPI_Barrier(gpuWorld);

      int nGlobalRows = A.M();
@@ -793,7 +804,7 @@ void AmgXSolver::SetMatrixMPITeams(const HypreParMatrix &A,

         AMGX_solver_setup(solver, AmgXA);

-         //Bind vectors to A
+         // Bind vectors to A
         AMGX_vector_bind(AmgXP, AmgXA);
         AMGX_vector_bind(AmgXRHS, AmgXA);
      }
@@ -847,11 +858,11 @@ void AmgXSolver::UpdateOperator(const Operator& op)

 void AmgXSolver::Mult(const Vector& B, Vector& X) const
 {
-   //Set intial guess to zero
+   // Set initial guess to zero
   X.UseDevice(true);
   X = 0.0;

-   //Mult for serial, and mpi-exclusive modes
+   // Mult for serial, and mpi-exclusive modes
   if (mpi_gpu_mode != "mpi-teams")
   {
      AMGX_vector_upload(AmgXP, X.Size(), 1, X.ReadWrite());
@@ -935,7 +946,7 @@ int AmgXSolver::GetNumIterations()

 void AmgXSolver::Finalize()
 {
-   //Check instance is initialized
+   // Check instance is initialized
   if (! isInitialized || count < 1)
   {
      mfem_error("Error in AmgXSolver::Finalize(). \n"
@@ -980,8 +991,8 @@ void AmgXSolver::Finalize()
 #endif
   }

-   // re-set necessary variables in case users want to reuse
-   // the variable of this instance for a new instance
+   // re-set necessary variables in case users want to reuse the variable of
+   // this instance for a new instance
 #ifdef MFEM_USE_MPI
   gpuProc = MPI_UNDEFINED;
   if (globalCpuWorld != MPI_COMM_NULL)
@@ -998,5 +1009,5 @@ void AmgXSolver::Finalize()
   isInitialized = false;
 }

-}//mfem namespace
+} // mfem namespace
 #endif
@@ -29,136 +29,115 @@ namespace mfem
 {

 /**
-   MFEM wrapper for Nvidia's multigrid library,
-   AmgX (https://github.com/NVIDIA/AMGX).
+   MFEM wrapper for Nvidia's multigrid library, AmgX (github.com/NVIDIA/AMGX)

-   AmgX requires building MFEM with CUDA, and AMGX
-   enabled. For distributed memory parallism,
-   MPI and Hypre (version 16.0+) are also required.
-   Although CUDA is required for building,
-   the AmgX wrapper is compatible with
-   a MFEM CPU device configuration.
+   AmgX requires building MFEM with CUDA, and AMGX enabled. For distributed
+   memory parallism, MPI and Hypre (version 16.0+) are also required. Although
+   CUDA is required for building, the AmgX solver is compatible with a MFEM CPU
+   device configuration.

-   The AmgXSolver class is designed to work
-   as a solver or preconditioner for MFEM existing
-   solvers. The AmgX solver class may be configured
-   in one of three ways:
+   The AmgXSolver class is designed to work as a solver or preconditioner for
+   existing MFEM solvers. The AmgX solver class may be configured in one of
+   three ways:

-   Serial - Takes a SparseMatrix solves
-   on a single GPU and assumes no MPI
+   Serial - Takes a SparseMatrix solves on a single GPU and assumes no MPI
   communication.

-   Exclusive GPU - Takes a HypreParMatrix
-   and assumes each MPI rank is paired with
-   an Nvidia GPU.
+   Exclusive GPU - Takes a HypreParMatrix and assumes each MPI rank is paired
+   with an Nvidia GPU.

-   MPI Teams - Takes a HypreParMatrix and
-   enables flexibility between number of MPI
-   ranks, and GPUs. Specifically, MPI ranks
-   are grouped with GPUs and a matrix consolidation
-   step is taken so the MPI root of each team
-   performs the necessary AmgX library calls.
-   The solution is then broadcasted to appropriate ranks.
-   This is particularly useful when configuring MFEM's
-   device as CPU.
+   MPI Teams - Takes a HypreParMatrix and enables flexibility between number of
+   MPI ranks, and GPUs. Specifically, MPI ranks are grouped with GPUs and a
+   matrix consolidation step is taken so the MPI root of each team performs the
+   necessary AmgX library calls. The solution is then broadcasted to appropriate
+   ranks. This is particularly useful when configuring MFEM's device as
+   CPU. This work is based on the AmgXWrapper of Chuang and Barba. Routines were
+   adopted and modified for setting up MPI communicators.

-   Examples 1,1p demonstrate basic usage with
-   default parameters, while examples under the
-   amgx folder demonstrate configuring the wrapper
-   as a solver, preconditioner, configuring and
-   running with exclusive GPU or MPI teams modes.
+   Examples 1/1p in the examples/amgx directory demonstrate configuring the
+   wrapper as a solver and preconditioner, as well as configuring and running
+   with exclusive GPU or MPI teams modes.

-   Reference:
-   Pi-Yueh Chuang, & Lorena A. Barba (2017).
-   AmgXWrapper: An interface between PETSc and the NVIDIA AmgX library.
-   J. Open Source Software, 2(16):280, doi:10.21105/joss.00280
+   This work is partially based on:

- */
+      Pi-Yueh Chuang and Lorena A. Barba (2017).
+      AmgXWrapper: An interface between PETSc and the NVIDIA AmgX library.
+      J. Open Source Software, 2(16):280, doi:10.21105/joss.00280
+
+   See https://github.com/barbagroup/AmgXWrapper.
+*/
 class AmgXSolver : public Solver
 {
 public:

-   /**
-     Flags to configure AmgXSolver
-     as a solver or preconditioner
-   */
+   /// Flags to configure AmgXSolver as a solver or preconditioner
   enum AMGX_MODE {SOLVER, PRECONDITIONER};

   /**
-     Flags to determine whether user solver
-     settings are defined internally
-     in the source code or will be read through an
-     external json file.
+      Flags to determine whether user solver settings are defined internally in
+      the source code or will be read through an external JSON file.
   */
   enum CONFIG_SRC {INTERNAL, EXTERNAL, UNDEFINED};

   AmgXSolver() = default;

   /**
-      Configures AmgX with a default configuration based on the
-      AmgX mode, and verbosity. Assumes no MPI parallism.
-    */
+      Configures AmgX with a default configuration based on the AmgX mode, and
+      verbosity. Assumes no MPI parallism.
+   */
   AmgXSolver(const AMGX_MODE amgxMode_, const bool verbose);

   /**
-     Once the solver configuration has been established through
-     either the ReadParameters method or the constructor,
-     InitSerial will initalize the library.
-     If configuring with constructor, the constructor will
-     make this call.
+      Once the solver configuration has been established through either the
+      ReadParameters method or the constructor, InitSerial will initalize the
+      library. If configuring with constructor, the constructor will make this
+      call.
   */
   void InitSerial();

 #ifdef MFEM_USE_MPI

   /**
-     Configures AmgX with a default configuration based on the
-     AmgX mode, and verbosity. Pairs each MPI rank with one GPU.
+      Configures AmgX with a default configuration based on the AmgX mode, and
+      verbosity. Pairs each MPI rank with one GPU.
   */
   AmgXSolver(const MPI_Comm &comm, const AMGX_MODE amgxMode_, const bool verbose);

   /**
-     Configures AmgX with a default configuration based on the
-     AmgX mode, and verbosity. Creates MPI teams around GPUs
-     to support MPI ranks > GPUs. Consolidates linear solver data
-     to avoid multiple ranks sharing GPUs. Requires specifying
-     number of devices in each compute node.
+      Configures AmgX with a default configuration based on the AmgX mode, and
+      verbosity. Creates MPI teams around GPUs to support MPI ranks >
+      GPUs. Consolidates linear solver data to avoid multiple ranks sharing
+      GPUs. Requires specifying number of devices in each compute node.
   */
   AmgXSolver(const MPI_Comm &comm, const int nDevs,
              const AMGX_MODE amgx_Mode_, const bool verbose);

   /**
-     Once the solver configuration has been established,
-     either through the constructor or the
-     ReadParameters method, InitSerial will initalize the library.
-     If configuring with constructor, the constructor will
-     make this call.
+      Once the solver configuration has been established, either through the
+      constructor or the ReadParameters method, InitSerial will initalize the
+      library. If configuring with constructor, the constructor will make this
+      call.
   */
   void InitExclusiveGPU(const MPI_Comm &comm);

   /**
-     Once the solver configuration has been established,
-     either through the ReadParameters method,
-     InitMPITeams will intialize the library and create
-     MPI teams based on the number of devices on each node
-     (nDevs).
-     If configuring with constructor, the constructor will
-     make this call.
+      Once the solver configuration has been established, either through the
+      ReadParameters method, InitMPITeams will initialize the library and create
+      MPI teams based on the number of devices on each node (nDevs). If
+      configuring with constructor, the constructor will make this call.
   */
   void InitMPITeams(const MPI_Comm &comm,
                     const int nDevs);
 #endif

   /**
-     Sets Operator for AmgX library, either
-     MFEM SparseMatrix or HypreParMatrix
+      Sets Operator for AmgX library, either MFEM SparseMatrix or HypreParMatrix
   */
   virtual void SetOperator(const Operator &op);

   /**
-     Replaces the matrix coefficients in the
-     AmgX solver.
-    */
+      Replaces the matrix coefficients in the AmgX solver.
+   */
   void UpdateOperator(const Operator &op);

   virtual void Mult(const Vector& b, Vector& x) const;
@@ -174,13 +153,12 @@ public:
      @param [in] verbose  true, false. Specifies the level
                           of verbosity.

-      When configured as a preconditioner, the default configuration
-      is to apply two iterations of an AMG V cycle with AmgX's default
-      smoother (block Jacobi).
+      When configured as a preconditioner, the default configuration is to apply
+      two iterations of an AMG V cycle with AmgX's default smoother (block
+      Jacobi).

-      As a solver the preconditioned conjugate gradient method is used.
-      The AMG V cycle with a block Jacobi smoother is used as a
-      preconditioner.
+      As a solver the preconditioned conjugate gradient method is used. The AMG
+      V-cycle with a block Jacobi smoother is used as a preconditioner.
   */
   void DefaultParameters(const AMGX_MODE amgxMode_, const bool verbose);

@@ -197,22 +175,22 @@ private:
   CONFIG_SRC configSrc = UNDEFINED;

 #ifdef MFEM_USE_MPI
-   // Consolidates matrix diagonal and off diagonal data
-   // and uploads matrix to AmgX.
+   // Consolidates matrix diagonal and off diagonal data and uploads matrix to
+   // AmgX.
   void SetMatrixMPIGPUExclusive(const HypreParMatrix &A,
                                 const Array<double> &loc_A,
                                 const Array<int> &loc_I, const Array<int64_t> &loc_J,
                                 const bool update_mat = false);

-   // Consolidates matrix diagonal and off diagonal data
-   // for all ranks in an MPI team. Root rank of each MPI
-   // team holds the the consolidated data and sets matrix.
+   // Consolidates matrix diagonal and off diagonal data for all ranks in an MPI
+   // team. Root rank of each MPI team holds the the consolidated data and sets
+   // matrix.
   void SetMatrixMPITeams(const HypreParMatrix &A, const Array<double> &loc_A,
                          const Array<int> &loc_I, const Array<int64_t> &loc_J,
                          const bool update_mat = false);

-   // The following methods consolidate array data to the root node in
-   // a MPI team.
+   // The following methods consolidate array data to the root node in a MPI
+   // team.
   void GatherArray(const Array<double> &inArr, Array<double> &outArr,
                    const int mpiTeamSz, const MPI_Comm &mpiTeam) const;

@@ -225,8 +203,8 @@ private:
   void GatherArray(const Array<int64_t> &inArr, Array<int64_t> &outArr,
                    const int mpiTeamSz, const MPI_Comm &mpiTeam) const;

-   // The following methods consolidate array data to the root node in
-   // a MPI team as well as store array partitions and displacements.
+   // The following methods consolidate array data to the root node in a MPI
+   // team as well as store array partitions and displacements.
   void GatherArray(const Vector &inArr, Vector &outArr,
                    const int mpiTeamSz, const MPI_Comm &mpiTeamComm,
                    Array<int> &Apart, Array<int> &Adisp) const;
@@ -275,20 +253,17 @@ private:

   int                     myGlobalRank;

-   // Total number of MPI procs in a node
-   // + rank id
+   // Total number of MPI procs in a node + rank id
   int                     localSize;

   int                     myLocalRank;

-   // Total number of MPI ranks sharing a device
-   // + rank id
+   // Total number of MPI ranks sharing a device + rank id
   int                     devWorldSize;

   int                     myDevWorldRank;

-   // Total number of MPI procs calling AmgX
-   // + rank id
+   // Total number of MPI procs calling AmgX + rank id
   int                     gpuWorldSize;

   int                     myGpuWorldRank;
@@ -328,11 +303,11 @@ private:

   void InitAmgX();

-   // Row partion for HypreMatrix
+   // Row partition for the HypreParMatrix
   int64_t mat_local_rows;

   std::string mpi_gpu_mode;
 };
 }
-#endif //MFEM_USE_AMGX
-#endif //MFEM_AMGX_SOLVER
+#endif // MFEM_USE_AMGX
+#endif // MFEM_AMGX_SOLVER
@@ -0,0 +1,225 @@
+#include "cpardiso.hpp"
+#include "hypre.hpp"
+#include <algorithm>
+#include <vector>
+#include <numeric>
+
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_MKL_CPARDISO
+
+namespace mfem
+{
+CPardisoSolver::CPardisoSolver(MPI_Comm comm) : comm_(comm)
+{
+   // Indicate that default parameters are changed
+   iparm[0] = 1;
+   // Use METIS for fill-in reordering
+   iparm[1] = 2;
+   // Do not write the solution into the x vector data
+   iparm[5] = 0;
+   // Maximum number of iterative refinement steps
+   iparm[7] = 2;
+   // Perturb the pivot elements with 1E-13
+   iparm[9] = 13;
+   // Use nonsymmetric permutation
+   iparm[10] = 1;
+   // Perform a check on the input data
+   iparm[26] = 1;
+   // 0-based indexing in CSR data structure
+   iparm[34] = 1;
+   // All inputs are distributed between MPI processes
+   iparm[39] = 2;
+   // Maximum number of numerical factorizations
+   maxfct = 1;
+   // Which factorization to use. This parameter is ignored and always assumed
+   // to be equal to 1. See MKL documentation.
+   mnum = 1;
+   // Print statistical information in file
+   msglvl = 0;
+   // Initialize error flag
+   error = 0;
+   // Real nonsymmetric matrix
+   mtype = MatType::REAL_NONSYMMETRIC;
+   // Number of right hand sides
+   nrhs = 1;
+};
+
+void CPardisoSolver::SetOperator(const Operator &op)
+{
+   auto hypreParMat = dynamic_cast<const HypreParMatrix &>(op);
+
+   MFEM_ASSERT(hypreParMat, "Must pass HypreParMatrix as Operator");
+
+   auto parcsr_op = static_cast<hypre_ParCSRMatrix *>(
+                       const_cast<HypreParMatrix &>(hypreParMat));
+
+   hypre_CSRMatrix *csr_op = hypre_MergeDiagAndOffd(parcsr_op);
+#if MFEM_HYPRE_VERSION >= 21600
+   hypre_CSRMatrixBigJtoJ(csr_op);
+#endif
+
+   m = parcsr_op->global_num_rows;
+   first_row = parcsr_op->first_row_index;
+   nnz_loc = csr_op->num_nonzeros;
+   m_loc = csr_op->num_rows;
+
+   height = m_loc;
+   width = m_loc;
+
+   double *csr_nzval = csr_op->data;
+   int *csr_colind = csr_op->j;
+
+   delete[] csr_rowptr;
+   delete[] reordered_csr_colind;
+   delete[] reordered_csr_nzval;
+   csr_rowptr = new int[m_loc + 1];
+   reordered_csr_colind = new int[nnz_loc];
+   reordered_csr_nzval = new double[nnz_loc];
+
+   for (int i = 0; i <= m_loc; i++)
+   {
+      csr_rowptr[i] = (csr_op->i)[i];
+   }
+
+   // CPardiso expects the column indices to be sorted for each row
+   std::vector<int> permutation_idx(nnz_loc);
+   std::iota(permutation_idx.begin(), permutation_idx.end(), 0);
+   for (int i = 0; i < m_loc; i++)
+   {
+      std::sort(permutation_idx.begin() + csr_rowptr[i],
+                permutation_idx.begin() + csr_rowptr[i + 1],
+                [csr_colind](int i1, int i2)
+      {
+         return csr_colind[i1] < csr_colind[i2];
+      });
+   }
+
+   for (int i = 0; i < nnz_loc; i++)
+   {
+      reordered_csr_colind[i] = csr_colind[permutation_idx[i]];
+      reordered_csr_nzval[i] = csr_nzval[permutation_idx[i]];
+   }
+
+   hypre_CSRMatrixDestroy(csr_op);
+
+   // The number of row in global matrix, rhs element and solution vector that
+   // begins the input domain belonging to this MPI process
+   iparm[40] = first_row;
+
+   // The number of row in global matrix, rhs element and solution vector that
+   // ends the input domain belonging to this MPI process
+   iparm[41] = first_row + m_loc - 1;
+
+   // Analyze inputs
+   phase = 11;
+   cluster_sparse_solver(pt,
+                         &maxfct,
+                         &mnum,
+                         &mtype,
+                         &phase,
+                         &m,
+                         reordered_csr_nzval,
+                         csr_rowptr,
+                         reordered_csr_colind,
+                         &idum,
+                         &nrhs,
+                         iparm,
+                         &msglvl,
+                         &ddum,
+                         &ddum,
+                         &comm_,
+                         &error);
+
+   MFEM_ASSERT(error == 0, "CPardiso analyze input error");
+
+   // Numerical factorization
+   phase = 22;
+   cluster_sparse_solver(pt,
+                         &maxfct,
+                         &mnum,
+                         &mtype,
+                         &phase,
+                         &m,
+                         reordered_csr_nzval,
+                         csr_rowptr,
+                         reordered_csr_colind,
+                         &idum,
+                         &nrhs,
+                         iparm,
+                         &msglvl,
+                         &ddum,
+                         &ddum,
+                         &comm_,
+                         &error);
+
+   MFEM_ASSERT(error == 0, "CPardiso factorization input error");
+}
+
+void CPardisoSolver::Mult(const Vector &b, Vector &x) const
+{
+   // Solve
+   phase = 33;
+   cluster_sparse_solver(pt,
+                         &maxfct,
+                         &mnum,
+                         &mtype,
+                         &phase,
+                         &m,
+                         reordered_csr_nzval,
+                         csr_rowptr,
+                         reordered_csr_colind,
+                         &idum,
+                         &nrhs,
+                         iparm,
+                         &msglvl,
+                         b.GetData(),
+                         x.GetData(),
+                         &comm_,
+                         &error);
+
+   MFEM_ASSERT(error == 0, "Pardiso solve error");
+}
+
+void CPardisoSolver::SetPrintLevel(int print_level)
+{
+   msglvl = print_level;
+}
+
+void CPardisoSolver::SetMatrixType(MatType mat_type)
+{
+   mtype = mat_type;
+}
+
+CPardisoSolver::~CPardisoSolver()
+{
+   // Release all internal memory
+   phase = -1;
+   cluster_sparse_solver(pt,
+                         &maxfct,
+                         &mnum,
+                         &mtype,
+                         &phase,
+                         &m,
+                         reordered_csr_nzval,
+                         csr_rowptr,
+                         reordered_csr_colind,
+                         &idum,
+                         &nrhs,
+                         iparm,
+                         &msglvl,
+                         &ddum,
+                         &ddum,
+                         &comm_,
+                         &error);
+
+   MFEM_ASSERT(error == 0, "CPardiso free error");
+
+   delete[] csr_rowptr;
+   delete[] reordered_csr_colind;
+   delete[] reordered_csr_nzval;
+}
+
+} // namespace mfem
+
+#endif // MFEM_USE_MKL_CPARDISO
+#endif // MFEM_USE_MPI
@@ -0,0 +1,125 @@
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_CPARDISO
+#define MFEM_CPARDISO
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+#ifdef MFEM_USE_MKL_CPARDISO
+
+#include "mkl_cluster_sparse_solver.h"
+#include "operator.hpp"
+
+namespace mfem
+{
+/**
+ * @brief MKL Parallel Direct Sparse Solver for Clusters
+ *
+ * Interface to MKL CPardiso: the MPI-enabled Intel MKL version of Pardiso
+ */
+class CPardisoSolver : public Solver
+{
+public:
+   enum MatType
+   {
+      REAL_STRUCTURE_SYMMETRIC = 1,
+      REAL_NONSYMMETRIC = 11
+   };
+
+   /**
+    * @brief Construct a new CPardisoSolver object
+    *
+    * @param comm MPI Communicator
+    */
+   CPardisoSolver(MPI_Comm comm);
+
+   /**
+    * @brief Set the Operator object and perform factorization
+    *
+    * @a op needs to be of type HypreParMatrix. The contents are copied and
+    * reordered in an internal CSR structure.
+    *
+    * @param op Operator to use in factorization and solve
+    */
+   void SetOperator(const Operator &op) override;
+
+   /**
+    * @brief Solve
+    *
+    * @param b RHS vector
+    * @param x Solution vector
+    */
+   void Mult(const Vector &b, Vector &x) const override;
+
+   /**
+    * @brief Set the print level for MKL CPardiso
+    *
+    * Prints statistics after the factorization and after each solve.
+    *
+    * @param print_lvl Print level
+    */
+   void SetPrintLevel(int print_lvl);
+
+   /**
+    * @brief Set the matrix type
+    *
+    * The matrix type supported is either real and symmetric or real and
+    * non-symmetric.
+    *
+    * @param mat_type Matrix type
+    */
+   void SetMatrixType(MatType mat_type);
+
+   ~CPardisoSolver();
+
+private:
+   MPI_Comm comm_;
+
+   // Global number of rows
+   int m;
+
+   // First row index of the global matrix on the local MPI rank
+   int first_row;
+
+   // Local number of nonzero entries
+   int nnz_loc;
+
+   // Local number of rows, obtained from a ParCSR matrix
+   int m_loc;
+
+   // CSR data structure for the copy data of the local CSR matrix
+   int *csr_rowptr = nullptr;
+   double *reordered_csr_nzval = nullptr;
+   int *reordered_csr_colind = nullptr;
+
+   // Internal solver memory pointer pt,
+   // 32-bit: int pt[64]
+   // 64-bit: long int pt[64] or void *pt[64] should be OK on both architectures
+   mutable void *pt[64] = {0};
+
+   // Solver control parameters, detailed description can be found in the
+   // constructor.
+   mutable int iparm[64] = {0};
+   mutable int maxfct, mnum, msglvl, phase, error;
+   int mtype;
+   int nrhs;
+
+   // Dummy variables
+   mutable int idum;
+   mutable double ddum;
+};
+} // namespace mfem
+
+#endif
+#endif // MFEM_USE_MKL_CPARDISO
+#endif // MFEM_USE_MPI
@@ -300,6 +300,43 @@ void OperatorHandle::EliminateRowsCols(OperatorHandle &A,
   }
 }

+void OperatorHandle::EliminateRows(const Array<int> &ess_dof_list)
+{
+   switch (Type())
+   {
+      case Operator::Hypre_ParCSR:
+      {
+#ifdef MFEM_USE_MPI
+         this->As<HypreParMatrix>()->EliminateRows(ess_dof_list);
+#else
+         MFEM_ABORT("type id = Hypre_ParCSR requires MFEM_USE_MPI");
+#endif
+         break;
+      }
+      default:
+         MFEM_ABORT(not_supported_msg << Type());
+   }
+}
+
+void OperatorHandle::EliminateCols(const Array<int> &ess_dof_list)
+{
+   switch (Type())
+   {
+      case Operator::Hypre_ParCSR:
+      {
+#ifdef MFEM_USE_MPI
+         auto Ae = this->As<HypreParMatrix>()->EliminateCols(ess_dof_list);
+         delete Ae;
+#else
+         MFEM_ABORT("type id = Hypre_ParCSR requires MFEM_USE_MPI");
+#endif
+         break;
+      }
+      default:
+         MFEM_ABORT(not_supported_msg << Type());
+   }
+}
+
 void OperatorHandle::EliminateBC(const OperatorHandle &A_e,
                                 const Array<int> &ess_dof_list,
                                 const Vector &X, Vector &B) const
@@ -186,6 +186,12 @@ public:
       elimination of the essential dofs @a ess_dof_list. */
   void EliminateRowsCols(OperatorHandle &A, const Array<int> &ess_dof_list);

+   /// Eliminate the rows corresponding to the essential dofs @a ess_dof_list
+   void EliminateRows(const Array<int> &ess_dof_list);
+
+   /// Eliminate columns corresponding to the essential dofs @a ess_dof_list
+   void EliminateCols(const Array<int> &ess_dof_list);
+
   /// Eliminate essential dofs from the solution @a X into the r.h.s. @a B.
   /** The argument @a A_e is expected to be the result of the method
       EliminateRowsCols(). */
@@ -2268,7 +2268,6 @@ void HypreSmoother::SetOperator(const Operator &op)
      Vector ones(height), diag(l1_norms, height);
      ones = 1.0;
      A->Mult(ones, diag);
-      type = 1;
   }
   else
   {
@@ -2420,13 +2419,17 @@ void HypreSmoother::Mult(const HypreParVector &b, HypreParVector &x) const
   }
   else
   {
+      int hypre_type = type;
+      // hypre doesn't have lumped Jacobi, so treat the action as l1-Jacobi
+      if (type == 5) { hypre_type = 1; }
+
      if (Z == NULL)
-         hypre_ParCSRRelax(*A, b, type,
+         hypre_ParCSRRelax(*A, b, hypre_type,
                           relax_times, l1_norms, relax_weight, omega,
                           max_eig_est, min_eig_est, poly_order, poly_fraction,
                           x, *V, NULL);
      else
-         hypre_ParCSRRelax(*A, b, type,
+         hypre_ParCSRRelax(*A, b, hypre_type,
                           relax_times, l1_norms, relax_weight, omega,
                           max_eig_est, min_eig_est, poly_order, poly_fraction,
                           x, *V, *Z);
@@ -2898,6 +2901,165 @@ HypreGMRES::~HypreGMRES()
 }


+HypreFGMRES::HypreFGMRES(MPI_Comm comm) : precond(NULL)
+{
+   iterative_mode = true;
+
+   HYPRE_ParCSRFlexGMRESCreate(comm, &fgmres_solver);
+   SetDefaultOptions();
+}
+
+HypreFGMRES::HypreFGMRES(HypreParMatrix &_A) : HypreSolver(&_A), precond(NULL)
+{
+   MPI_Comm comm;
+
+   iterative_mode = true;
+
+   HYPRE_ParCSRMatrixGetComm(*A, &comm);
+
+   HYPRE_ParCSRFlexGMRESCreate(comm, &fgmres_solver);
+   SetDefaultOptions();
+}
+
+void HypreFGMRES::SetDefaultOptions()
+{
+   int k_dim    = 50;
+   int max_iter = 100;
+   double tol   = 1e-6;
+
+   HYPRE_ParCSRFlexGMRESSetKDim(fgmres_solver, k_dim);
+   HYPRE_ParCSRFlexGMRESSetMaxIter(fgmres_solver, max_iter);
+   HYPRE_ParCSRFlexGMRESSetTol(fgmres_solver, tol);
+}
+
+void HypreFGMRES::SetOperator(const Operator &op)
+{
+   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
+   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
+
+   // update base classes: Operator, Solver, HypreSolver
+   height = new_A->Height();
+   width  = new_A->Width();
+   A = const_cast<HypreParMatrix *>(new_A);
+   if (precond)
+   {
+      precond->SetOperator(*A);
+      this->SetPreconditioner(*precond);
+   }
+   setup_called = 0;
+   delete X;
+   delete B;
+   B = X = NULL;
+}
+
+void HypreFGMRES::SetTol(double tol)
+{
+   HYPRE_ParCSRFlexGMRESSetTol(fgmres_solver, tol);
+}
+
+void HypreFGMRES::SetMaxIter(int max_iter)
+{
+   HYPRE_ParCSRFlexGMRESSetMaxIter(fgmres_solver, max_iter);
+}
+
+void HypreFGMRES::SetKDim(int k_dim)
+{
+   HYPRE_ParCSRFlexGMRESSetKDim(fgmres_solver, k_dim);
+}
+
+void HypreFGMRES::SetLogging(int logging)
+{
+   HYPRE_ParCSRFlexGMRESSetLogging(fgmres_solver, logging);
+}
+
+void HypreFGMRES::SetPrintLevel(int print_lvl)
+{
+   HYPRE_ParCSRFlexGMRESSetPrintLevel(fgmres_solver, print_lvl);
+}
+
+void HypreFGMRES::SetPreconditioner(HypreSolver &_precond)
+{
+   precond = &_precond;
+   HYPRE_ParCSRFlexGMRESSetPrecond(fgmres_solver,
+                                   _precond.SolveFcn(),
+                                   _precond.SetupFcn(),
+                                   _precond);
+}
+
+void HypreFGMRES::Mult(const HypreParVector &b, HypreParVector &x) const
+{
+   int myid;
+   HYPRE_Int time_index = 0;
+   HYPRE_Int num_iterations;
+   double final_res_norm;
+   MPI_Comm comm;
+   HYPRE_Int print_level;
+
+   HYPRE_FlexGMRESGetPrintLevel(fgmres_solver, &print_level);
+
+   HYPRE_ParCSRMatrixGetComm(*A, &comm);
+
+   if (!setup_called)
+   {
+      if (print_level > 0)
+      {
+         time_index = hypre_InitializeTiming("FGMRES Setup");
+         hypre_BeginTiming(time_index);
+      }
+
+      HYPRE_ParCSRFlexGMRESSetup(fgmres_solver, *A, b, x);
+      setup_called = 1;
+
+      if (print_level > 0)
+      {
+         hypre_EndTiming(time_index);
+         hypre_PrintTiming("Setup phase times", comm);
+         hypre_FinalizeTiming(time_index);
+         hypre_ClearTiming();
+      }
+   }
+
+   if (print_level > 0)
+   {
+      time_index = hypre_InitializeTiming("FGMRES Solve");
+      hypre_BeginTiming(time_index);
+   }
+
+   if (!iterative_mode)
+   {
+      x = 0.0;
+   }
+
+   HYPRE_ParCSRFlexGMRESSolve(fgmres_solver, *A, b, x);
+
+   if (print_level > 0)
+   {
+      hypre_EndTiming(time_index);
+      hypre_PrintTiming("Solve phase times", comm);
+      hypre_FinalizeTiming(time_index);
+      hypre_ClearTiming();
+
+      HYPRE_ParCSRFlexGMRESGetNumIterations(fgmres_solver, &num_iterations);
+      HYPRE_ParCSRFlexGMRESGetFinalRelativeResidualNorm(fgmres_solver,
+                                                        &final_res_norm);
+
+      MPI_Comm_rank(comm, &myid);
+
+      if (myid == 0)
+      {
+         mfem::out << "FGMRES Iterations = " << num_iterations << endl
+                   << "Final FGMRES Relative Residual Norm = " << final_res_norm
+                   << endl;
+      }
+   }
+}
+
+HypreFGMRES::~HypreFGMRES()
+{
+   HYPRE_ParCSRFlexGMRESDestroy(fgmres_solver);
+}
+
+
 void HypreDiagScale::SetOperator(const Operator &op)
 {
   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
@@ -3080,6 +3242,84 @@ HypreEuclid::~HypreEuclid()
 }


+#if MFEM_HYPRE_VERSION >= 21900
+HypreILU::HypreILU()
+{
+   HYPRE_ILUCreate(&ilu_precond);
+   SetDefaultOptions();
+}
+
+void HypreILU::SetDefaultOptions()
+{
+   // The type of incomplete LU used locally and globally (see class doc)
+   HYPRE_Int ilu_type = 0; // ILU(k) locally and block Jacobi globally
+   HYPRE_ILUSetType(ilu_precond, ilu_type);
+
+   // Maximum iterations; 1 iter for preconditioning
+   HYPRE_Int max_iter = 1;
+   HYPRE_ILUSetMaxIter(ilu_precond, max_iter);
+
+   // The tolerance when used as a smoother; set to 0.0 for preconditioner
+   HYPRE_Real tol = 0.0;
+   HYPRE_ILUSetTol(ilu_precond, tol);
+
+   // Fill level for ILU(k)
+   HYPRE_Int lev_fill = 1;
+   HYPRE_ILUSetLevelOfFill(ilu_precond, lev_fill);
+
+   // Local reordering scheme; 0 = no reordering, 1 = reverse Cuthill-McKee
+   HYPRE_Int reorder_type = 1;
+   HYPRE_ILUSetLocalReordering(ilu_precond, reorder_type);
+
+   // Information print level; 0 = none, 1 = setup, 2 = solve, 3 = setup+solve
+   HYPRE_Int print_level = 0;
+   HYPRE_ILUSetPrintLevel(ilu_precond, print_level);
+}
+
+void HypreILU::ResetILUPrecond()
+{
+   if (ilu_precond)
+   {
+      HYPRE_ILUDestroy(ilu_precond);
+   }
+   HYPRE_ILUCreate(&ilu_precond);
+   SetDefaultOptions();
+}
+
+void HypreILU::SetLevelOfFill(HYPRE_Int lev_fill)
+{
+   HYPRE_ILUSetLevelOfFill(ilu_precond, lev_fill);
+}
+
+void HypreILU::SetPrintLevel(HYPRE_Int print_level)
+{
+   HYPRE_ILUSetPrintLevel(ilu_precond, print_level);
+}
+
+void HypreILU::SetOperator(const Operator &op)
+{
+   const HypreParMatrix *new_A = dynamic_cast<const HypreParMatrix *>(&op);
+   MFEM_VERIFY(new_A, "new Operator must be a HypreParMatrix!");
+
+   if (A) { ResetILUPrecond(); }
+
+   // update base classes: Operator, Solver, HypreSolver
+   height = new_A->Height();
+   width  = new_A->Width();
+   A = const_cast<HypreParMatrix *>(new_A);
+   setup_called = 0;
+   delete X;
+   delete B;
+   B = X = NULL;
+}
+
+HypreILU::~HypreILU()
+{
+   HYPRE_ILUDestroy(ilu_precond);
+}
+#endif
+
+
 HypreBoomerAMG::HypreBoomerAMG()
 {
   HYPRE_BoomerAMGCreate(&amg_precond);
@@ -160,9 +160,10 @@ public:
   ~HypreParVector();

 #ifdef MFEM_USE_SUNDIALS
-   /// Return a new wrapper SUNDIALS N_Vector of type SUNDIALS_NVEC_PARALLEL.
-   /** The returned N_Vector must be destroyed by the caller. */
-   virtual N_Vector ToNVector();
+   /// (DEPRECATED) Return a new wrapper SUNDIALS N_Vector of type SUNDIALS_NVEC_PARALLEL.
+   /** @deprecated The returned N_Vector must be destroyed by the caller. */
+   MFEM_DEPRECATED virtual N_Vector ToNVector();
+   using Vector::ToNVector;
 #endif
 };

@@ -782,8 +783,11 @@ public:
       2) enable residual-based stopping criteria. */
   void SetResidualConvergenceOptions(int res_frequency=-1, double rtol=0.0);

+   /// deprecated: use SetZeroInitialIterate()
+   MFEM_DEPRECATED void SetZeroInintialIterate() { iterative_mode = false; }
+
   /// non-hypre setting
-   void SetZeroInintialIterate() { iterative_mode = false; }
+   void SetZeroInitialIterate() { iterative_mode = false; }

   void GetNumIterations(int &num_iterations)
   {
@@ -836,8 +840,11 @@ public:
   /// Set the hypre solver to be used as a preconditioner
   void SetPreconditioner(HypreSolver &precond);

+   /// deprecated: use SetZeroInitialIterate()
+   MFEM_DEPRECATED void SetZeroInintialIterate() { iterative_mode = false; }
+
   /// non-hypre setting
-   void SetZeroInintialIterate() { iterative_mode = false; }
+   void SetZeroInitialIterate() { iterative_mode = false; }

   /// The typecast to HYPRE_Solver returns the internal gmres_solver
   virtual operator HYPRE_Solver() const  { return gmres_solver; }
@@ -856,6 +863,56 @@ public:
   virtual ~HypreGMRES();
 };

+/// Flexible GMRES solver in hypre
+class HypreFGMRES : public HypreSolver
+{
+private:
+   HYPRE_Solver fgmres_solver;
+
+   HypreSolver * precond;
+
+   /// Default, generally robust, FGMRES options
+   void SetDefaultOptions();
+
+public:
+   HypreFGMRES(MPI_Comm comm);
+
+   HypreFGMRES(HypreParMatrix &_A);
+
+   virtual void SetOperator(const Operator &op);
+
+   void SetTol(double tol);
+   void SetMaxIter(int max_iter);
+   void SetKDim(int dim);
+   void SetLogging(int logging);
+   void SetPrintLevel(int print_lvl);
+
+   /// Set the hypre solver to be used as a preconditioner
+   void SetPreconditioner(HypreSolver &precond);
+
+   /// deprecated: use SetZeroInitialIterate()
+   MFEM_DEPRECATED void SetZeroInintialIterate() { iterative_mode = false; }
+
+   /// non-hypre setting
+   void SetZeroInitialIterate() { iterative_mode = false; }
+
+   /// The typecast to HYPRE_Solver returns the internal fgmres_solver
+   virtual operator HYPRE_Solver() const  { return fgmres_solver; }
+
+   /// FGMRES Setup function
+   virtual HYPRE_PtrToParSolverFcn SetupFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_ParCSRFlexGMRESSetup; }
+   /// FGMRES Solve function
+   virtual HYPRE_PtrToParSolverFcn SolveFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_ParCSRFlexGMRESSolve; }
+
+   /// Solve Ax=b with hypre's FGMRES
+   virtual void Mult (const HypreParVector &b, HypreParVector &x) const;
+   using HypreSolver::Mult;
+
+   virtual ~HypreFGMRES();
+};
+
 /// The identity operator as a hypre solver
 class HypreIdentity : public HypreSolver
 {
@@ -962,6 +1019,62 @@ public:
   virtual ~HypreEuclid();
 };

+#if MFEM_HYPRE_VERSION >= 21900
+/**
+@brief Wrapper for Hypre's native parallel ILU preconditioner.
+
+The default ILU factorization type is ILU(k).  If you need to change this, or
+any other option, you can use the HYPRE_Solver method to cast the object for use
+with Hypre's native functions. For example, if want to use natural ordering
+rather than RCM reordering, you can use the following approach:
+
+@code
+mfem::HypreILU ilu();
+int reorder_type = 0;
+HYPRE_ILUSetLocalReordering(ilu, reorder_type);
+@endcode
+*/
+class HypreILU : public HypreSolver
+{
+private:
+   HYPRE_Solver ilu_precond;
+
+   /// Set the ILU default options
+   void SetDefaultOptions();
+
+   /** Reset the ILU preconditioner.
+   @note If ilu_precond is NULL, this method allocates; otherwise it destroys
+   ilu_precond and allocates a new object.  In both cases the default options
+   are set. */
+   void ResetILUPrecond();
+
+public:
+   /// Constructor; sets the default options
+   HypreILU();
+
+   virtual ~HypreILU();
+
+   /// Set the fill level for ILU(k); the default is k=1.
+   void SetLevelOfFill(HYPRE_Int lev_fill);
+
+   /// Set the print level: 0 = none, 1 = setup, 2 = solve, 3 = setup+solve
+   void SetPrintLevel(HYPRE_Int print_level);
+
+   /// The typecast to HYPRE_Solver returns the internal ilu_precond
+   virtual operator HYPRE_Solver() const { return ilu_precond; }
+
+   virtual void SetOperator(const Operator &op);
+
+   /// ILU Setup function
+   virtual HYPRE_PtrToParSolverFcn SetupFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_ILUSetup; }
+
+   /// ILU Solve function
+   virtual HYPRE_PtrToParSolverFcn SolveFcn() const
+   { return (HYPRE_PtrToParSolverFcn) HYPRE_ILUSolve; }
+};
+#endif
+
 /// The BoomerAMG solver in hypre
 class HypreBoomerAMG : public HypreSolver
 {
@@ -65,6 +65,10 @@
 #include "strumpack.hpp"
 #endif

+#ifdef MFEM_USE_MKL_CPARDISO
+#include "cpardiso.hpp"
+#endif
+
 #endif // MFEM_USE_MPI

 #endif
--- a/Show More
+++ b/Show More